From b7c652ab07562f3beb40239901cb6b5a3f33f09b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 9 Mar 2020 18:23:44 +0200 Subject: [PATCH 001/196] Refactoring of Queue Interfaces --- .../thingsboard/server/TbQueueCallback.java | 8 ++++ .../thingsboard/server/TbQueueConsumer.java | 11 +++++ .../org/thingsboard/server/TbQueueMsg.java | 13 ++++++ .../server/TbQueueMsgMetadata.java | 4 ++ .../thingsboard/server/TbQueueProducer.java | 9 ++++ .../server/TbQueueRequestTemplate.java | 9 ++++ .../transport/queue/TransportApiCall.java | 31 +++++++++++++ .../queue/TransportApiCallRequest.java | 40 +++++++++++++++++ .../queue/TransportQueueProvider.java | 19 ++++++++ .../service/AbstractTransportService.java | 5 +++ .../src/main/resources/tb-mqtt-transport.yml | 43 +++++++++---------- 11 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java new file mode 100644 index 0000000000..a1b64bd205 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -0,0 +1,8 @@ +package org.thingsboard.server; + +public interface TbQueueCallback { + + void onSuccess(); + + void onFailure(Throwable t); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java new file mode 100644 index 0000000000..02010522e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -0,0 +1,11 @@ +package org.thingsboard.server; + +import java.util.List; + +public interface TbQueueConsumer { + + List poll(); + + void commit(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java new file mode 100644 index 0000000000..4f0e5b7f2e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -0,0 +1,13 @@ +package org.thingsboard.server; + +import java.util.Map; +import java.util.UUID; + +public interface TbQueueMsg { + + UUID getKey(); + + Map getHeaders(); + + byte[] getData(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java new file mode 100644 index 0000000000..8eecae51fb --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java @@ -0,0 +1,4 @@ +package org.thingsboard.server; + +public interface TbQueueMsgMetadata { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java new file mode 100644 index 0000000000..a36deda8e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -0,0 +1,9 @@ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueProducer { + + ListenableFuture send(T msg, TbQueueCallback callback); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java new file mode 100644 index 0000000000..a820d5fcb4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -0,0 +1,9 @@ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueRequestTemplate { + + ListenableFuture post(String key, Request request); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java new file mode 100644 index 0000000000..956c1138f1 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java @@ -0,0 +1,31 @@ +package org.thingsboard.server.common.transport.queue; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public abstract class TransportApiCall { + + protected byte[] uuidToBytes(UUID uuid) { + ByteBuffer buf = ByteBuffer.allocate(16); + buf.putLong(uuid.getMostSignificantBits()); + buf.putLong(uuid.getLeastSignificantBits()); + return buf.array(); + } + + protected static UUID bytesToUuid(byte[] bytes) { + ByteBuffer bb = ByteBuffer.wrap(bytes); + long firstLong = bb.getLong(); + long secondLong = bb.getLong(); + return new UUID(firstLong, secondLong); + } + + protected byte[] stringToBytes(String string) { + return string.getBytes(StandardCharsets.UTF_8); + } + + protected String bytesToString(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java new file mode 100644 index 0000000000..49b83e2c17 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java @@ -0,0 +1,40 @@ +package org.thingsboard.server.common.transport.queue; + +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class TransportApiCallRequest extends TransportApiCall implements TbQueueMsg { + public static final String REQUEST_ID_HEADER = "requestId"; + public static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + + private final UUID requestId; + private final Map headers; + private final TransportProtos.TransportApiRequestMsg msg; + + public TransportApiCallRequest(UUID requestId, String responseTopic, TransportProtos.TransportApiRequestMsg msg) { + this.requestId = requestId; + this.headers = new HashMap<>(); + this.headers.put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + this.headers.put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTopic)); + this.msg = msg; + } + + @Override + public UUID getKey() { + return requestId; + } + + @Override + public Map getHeaders() { + return null; + } + + @Override + public byte[] getData() { + return msg.toByteArray(); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java new file mode 100644 index 0000000000..d4ce11afec --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java @@ -0,0 +1,19 @@ +package org.thingsboard.server.common.transport.queue; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueProducer; + +public interface TransportQueueProvider { + + TbQueueProducer getTransportApiCallRequestsProducer(); + + TbQueueConsumer getTransportApiCallResponsesConsumer(); + + TbQueueProducer getRuleEngineMsgProducer(); + + TbQueueProducer getTbCoreMsgProducer(); + + TbQueueConsumer getTransportNotificationsConsumer(); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java index 77c45e1714..0aaa8cef68 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; @@ -26,6 +27,7 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.queue.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Random; @@ -49,6 +51,9 @@ public abstract class AbstractTransportService implements TransportService { @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; + @Autowired + private TransportQueueProvider queueProvider; + protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index d13a3d071e..47dfe57c85 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -61,25 +61,24 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + queue: + type: kafka + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + transport_api: + requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule_engine: + topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" + notifications: + topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" From a39e8c37567ba8a1e07d78dced5a22f876c4d644 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 10 Mar 2020 12:34:05 +0200 Subject: [PATCH 002/196] Queue Implementation --- .../org/thingsboard/server/TbQueueAdmin.java | 9 + .../thingsboard/server/TbQueueCallback.java | 2 +- .../thingsboard/server/TbQueueConsumer.java | 6 +- .../thingsboard/server/TbQueueHandler.java | 27 +++ .../org/thingsboard/server/TbQueueMsg.java | 3 +- .../thingsboard/server/TbQueueMsgHeaders.java | 8 + .../thingsboard/server/TbQueueProducer.java | 6 + .../server/TbQueueRequestTemplate.java | 2 +- .../server/TbQueueResponseTemplate.java | 8 + .../common/AbstractTbQueueTemplate.java | 32 ++++ .../server/common/AsyncCallbackTemplate.java | 66 +++++++ .../common/DefaultTbQueueRequestTemplate.java | 172 ++++++++++++++++++ .../DefaultTbQueueResponseTemplate.java | 137 ++++++++++++++ 13 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java new file mode 100644 index 0000000000..7f8cff5f22 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -0,0 +1,9 @@ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +public interface TbQueueAdmin { + + ListenableFuture createTopicIfNotExists(String topic); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java index a1b64bd205..3d7d791ae4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -2,7 +2,7 @@ package org.thingsboard.server; public interface TbQueueCallback { - void onSuccess(); + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index 02010522e4..181d50bdd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -4,7 +4,11 @@ import java.util.List; public interface TbQueueConsumer { - List poll(); + String getTopic(); + + void subscribe(); + + List poll(long durationInMillis); void commit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java new file mode 100644 index 0000000000..93004cddca --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import com.google.common.util.concurrent.ListenableFuture; + +/** + * Created by ashvayka on 05.10.18. + */ +public interface TbQueueHandler { + + ListenableFuture handle(Request request); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java index 4f0e5b7f2e..22714af77e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -1,13 +1,12 @@ package org.thingsboard.server; -import java.util.Map; import java.util.UUID; public interface TbQueueMsg { UUID getKey(); - Map getHeaders(); + TbQueueMsgHeaders getHeaders(); byte[] getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java new file mode 100644 index 0000000000..e1a5b4861d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -0,0 +1,8 @@ +package org.thingsboard.server; + +public interface TbQueueMsgHeaders { + + byte[] put(String key, byte[] value); + + byte[] get(String key); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index a36deda8e4..41d165d743 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -4,6 +4,12 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueProducer { + void init(); + + String getDefaultTopic(); + ListenableFuture send(T msg, TbQueueCallback callback); + ListenableFuture send(String topic, T msg, TbQueueCallback callback); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index a820d5fcb4..85b5cc3f15 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -4,6 +4,6 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueRequestTemplate { - ListenableFuture post(String key, Request request); + ListenableFuture send(String key, Request request); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java new file mode 100644 index 0000000000..f3b4e4ad5d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java @@ -0,0 +1,8 @@ +package org.thingsboard.server; + +public interface TbQueueResponseTemplate { + + void init(TbQueueHandler handler); + + void stop(); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java new file mode 100644 index 0000000000..2f406a9631 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -0,0 +1,32 @@ +package org.thingsboard.server.common; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public class AbstractTbQueueTemplate { + protected static final String REQUEST_ID_HEADER = "requestId"; + protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + + protected byte[] uuidToBytes(UUID uuid) { + ByteBuffer buf = ByteBuffer.allocate(16); + buf.putLong(uuid.getMostSignificantBits()); + buf.putLong(uuid.getLeastSignificantBits()); + return buf.array(); + } + + protected static UUID bytesToUuid(byte[] bytes) { + ByteBuffer bb = ByteBuffer.wrap(bytes); + long firstLong = bb.getLong(); + long secondLong = bb.getLong(); + return new UUID(firstLong, secondLong); + } + + protected byte[] stringToBytes(String string) { + return string.getBytes(StandardCharsets.UTF_8); + } + + protected String bytesToString(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java new file mode 100644 index 0000000000..95783f87cd --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 05.10.18. + */ +public class AsyncCallbackTemplate { + + public static void withCallbackAndTimeout(ListenableFuture future, + Consumer onSuccess, + Consumer onFailure, + long timeoutInMs, + ScheduledExecutorService timeoutExecutor, + Executor callbackExecutor) { + future = Futures.withTimeout(future, timeoutInMs, TimeUnit.MILLISECONDS, timeoutExecutor); + withCallback(future, onSuccess, onFailure, callbackExecutor); + } + + public static void withCallback(ListenableFuture future, Consumer onSuccess, + Consumer onFailure, Executor executor) { + FutureCallback callback = new FutureCallback() { + @Override + public void onSuccess(T result) { + try { + onSuccess.accept(result); + } catch (Throwable th) { + onFailure(th); + } + } + + @Override + public void onFailure(Throwable t) { + onFailure.accept(t); + } + }; + if (executor != null) { + Futures.addCallback(future, callback, executor); + } else { + Futures.addCallback(future, callback); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java new file mode 100644 index 0000000000..2965b7a291 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -0,0 +1,172 @@ +package org.thingsboard.server.common; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.InterruptException; +import org.thingsboard.server.TbQueueAdmin; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class DefaultTbQueueRequestTemplate extends AbstractTbQueueTemplate + implements TbQueueRequestTemplate { + + private final TbQueueAdmin queueAdmin; + private final TbQueueProducer requestTemplate; + private final TbQueueConsumer responseTemplate; + private final ConcurrentMap> pendingRequests; + private final boolean internalExecutor; + private final ExecutorService executor; + private final long maxRequestTimeout; + private final long maxPendingRequests; + private final long pollInterval; + private volatile long tickTs = 0L; + private volatile long tickSize = 0L; + private volatile boolean stopped = false; + + @Builder + public DefaultTbQueueRequestTemplate(TbQueueAdmin queueAdmin, + TbQueueProducer requestTemplate, + TbQueueConsumer responseTemplate, + long maxRequestTimeout, + long maxPendingRequests, + long pollInterval, + ExecutorService executor) { + this.queueAdmin = queueAdmin; + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxRequestTimeout = maxRequestTimeout; + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + if (executor != null) { + internalExecutor = false; + this.executor = executor; + } else { + internalExecutor = true; + this.executor = Executors.newSingleThreadExecutor(); + } + } + + public void init() { + queueAdmin.createTopicIfNotExists(responseTemplate.getTopic()); + this.requestTemplate.init(); + tickTs = System.currentTimeMillis(); + responseTemplate.subscribe(); + executor.submit(() -> { + long nextCleanupMs = 0L; + while (!stopped) { + try { + List responses = responseTemplate.poll(pollInterval); + if (responses.size() > 0) { + log.trace("Polling responses completed, consumer records count [{}]", responses.size()); + } + responses.forEach(response -> { + log.trace("Received response to Kafka Template request: {}", response); + byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); + UUID requestId = null; + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header and body", response); + } else { + requestId = bytesToUuid(requestIdHeader); + log.trace("[{}] Response received", requestId); + ResponseMetaData expectedResponse = pendingRequests.remove(requestId); + if (expectedResponse == null) { + log.trace("[{}] Invalid or stale request", requestId); + } else { + expectedResponse.future.set(response); + } + } + }); + responseTemplate.commit(); + tickTs = System.currentTimeMillis(); + tickSize = pendingRequests.size(); + if (nextCleanupMs < tickTs) { + //cleanup; + pendingRequests.forEach((key, value) -> { + if (value.expTime < tickTs) { + ResponseMetaData staleRequest = pendingRequests.remove(key); + if (staleRequest != null) { + log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); + staleRequest.future.setException(new TimeoutException()); + } + } + }); + nextCleanupMs = tickTs + maxRequestTimeout; + } + } catch (InterruptException ie) { + if (!stopped) { + log.warn("Fetching data from kafka was interrupted.", ie); + } + } catch (Throwable e) { + log.warn("Failed to obtain responses from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new responses", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (internalExecutor) { + executor.shutdownNow(); + } + } + + @Override + public ListenableFuture send(String key, Request request) { + if (tickSize > maxPendingRequests) { + return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); + } + UUID requestId = UUID.randomUUID(); + request.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + request.getHeaders().put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())); + SettableFuture future = SettableFuture.create(); + ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); + pendingRequests.putIfAbsent(requestId, responseMetaData); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); + requestTemplate.send(request, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("[{}] Request sent: {}", requestId, metadata); + } + + @Override + public void onFailure(Throwable t) { + pendingRequests.remove(requestId); + future.setException(t); + } + }); + return future; + } + + private static class ResponseMetaData { + private final long expTime; + private final SettableFuture future; + + ResponseMetaData(long ts, SettableFuture future) { + this.expTime = ts; + this.future = future; + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java new file mode 100644 index 0000000000..92dac0551b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -0,0 +1,137 @@ +package org.thingsboard.server.common; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.InterruptException; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueHandler; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueResponseTemplate; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DefaultTbQueueResponseTemplate extends AbstractTbQueueTemplate + implements TbQueueResponseTemplate { + + private final TbQueueConsumer requestTemplate; + private final TbQueueProducer responseTemplate; + private final ConcurrentMap pendingRequests; + private final ExecutorService loopExecutor; + private final ScheduledExecutorService timeoutExecutor; + private final ExecutorService callbackExecutor; + private final int maxPendingRequests; + private final long requestTimeout; + + private final long pollInterval; + private volatile boolean stopped = false; + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public DefaultTbQueueResponseTemplate(TbQueueConsumer requestTemplate, + TbQueueProducer responseTemplate, + TbQueueHandler handler, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService executor) { + this.requestTemplate = requestTemplate; + this.responseTemplate = responseTemplate; + this.pendingRequests = new ConcurrentHashMap<>(); + this.maxPendingRequests = maxPendingRequests; + this.pollInterval = pollInterval; + this.requestTimeout = requestTimeout; + this.callbackExecutor = executor; + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); + this.loopExecutor = Executors.newSingleThreadExecutor(); + } + + @Override + public void init(TbQueueHandler handler) { + this.responseTemplate.init(); + requestTemplate.subscribe(); + loopExecutor.submit(() -> { + while (!stopped) { + try { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + List requests = requestTemplate.poll(pollInterval); + requests.forEach(request -> { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseTemplate.send(responseTopic, response, null); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } + }); + requestTemplate.commit(); + } catch (InterruptException ie) { + if (!stopped) { + log.warn("Fetching data from queue was interrupted.", ie); + } + } catch (Throwable e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + public void stop() { + stopped = true; + if (timeoutExecutor != null) { + timeoutExecutor.shutdownNow(); + } + if (loopExecutor != null) { + loopExecutor.shutdownNow(); + } + } + +} From 7b2bdeab236f75b16f246c844665eaeea7162820 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 10 Mar 2020 14:03:06 +0200 Subject: [PATCH 003/196] Refactoring of Transport Interfaces --- common/queue/pom.xml | 4 + .../server/TbQueueRequestTemplate.java | 2 +- .../common/DefaultTbQueueMsgHeaders.java | 21 ++ .../common/DefaultTbQueueRequestTemplate.java | 4 +- .../server/common/TbProtoQueueMsg.java | 40 ++++ .../queue/KafkaTransportQueueProvider.java | 31 +++ .../transport/queue/TransportApiCall.java | 31 --- .../queue/TransportApiCallRequest.java | 40 ---- .../queue/TransportQueueProvider.java | 17 +- .../service/AbstractTransportService.java | 27 ++- .../service/RemoteTransportService.java | 200 +++++++++--------- 11 files changed, 234 insertions(+), 183 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 8f0a0913a4..952b3c540d 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -94,6 +94,10 @@ mockito-all test + + com.google.protobuf + protobuf-java + diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 85b5cc3f15..5ba28f22c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -4,6 +4,6 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueRequestTemplate { - ListenableFuture send(String key, Request request); + ListenableFuture send(Request request); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java new file mode 100644 index 0000000000..4ef33a489d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -0,0 +1,21 @@ +package org.thingsboard.server.common; + +import org.thingsboard.server.TbQueueMsgHeaders; + +import java.util.HashMap; +import java.util.Map; + +public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { + + private final Map data = new HashMap<>(); + + @Override + public byte[] put(String key, byte[] value) { + return data.put(key, value); + } + + @Override + public byte[] get(String key) { + return data.get(key); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 2965b7a291..49022ad2e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -133,7 +133,7 @@ public class DefaultTbQueueRequestTemplate send(String key, Request request) { + public ListenableFuture send(Request request) { if (tickSize > maxPendingRequests) { return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); } @@ -143,7 +143,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); requestTemplate.send(request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java new file mode 100644 index 0000000000..f1120b876f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -0,0 +1,40 @@ +package org.thingsboard.server.common; + +import lombok.Data; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgHeaders; + +import java.util.UUID; + +@Data +public class TbProtoQueueMsg implements TbQueueMsg { + + private final UUID key; + private final T value; + private final DefaultTbQueueMsgHeaders headers; + + public TbProtoQueueMsg(UUID key, T value) { + this(key, value, new DefaultTbQueueMsgHeaders()); + } + + public TbProtoQueueMsg(UUID key, T value, DefaultTbQueueMsgHeaders headers) { + this.key = key; + this.value = value; + this.headers = headers; + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return value.toByteArray(); + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java new file mode 100644 index 0000000000..2aa6f90b7b --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java @@ -0,0 +1,31 @@ +package org.thingsboard.server.common.transport.queue; + +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +@Component +public class KafkaTransportQueueProvider implements TransportQueueProvider { + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + return null; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return null; + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java deleted file mode 100644 index 956c1138f1..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCall.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.thingsboard.server.common.transport.queue; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -public abstract class TransportApiCall { - - protected byte[] uuidToBytes(UUID uuid) { - ByteBuffer buf = ByteBuffer.allocate(16); - buf.putLong(uuid.getMostSignificantBits()); - buf.putLong(uuid.getLeastSignificantBits()); - return buf.array(); - } - - protected static UUID bytesToUuid(byte[] bytes) { - ByteBuffer bb = ByteBuffer.wrap(bytes); - long firstLong = bb.getLong(); - long secondLong = bb.getLong(); - return new UUID(firstLong, secondLong); - } - - protected byte[] stringToBytes(String string) { - return string.getBytes(StandardCharsets.UTF_8); - } - - protected String bytesToString(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } - -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java deleted file mode 100644 index 49b83e2c17..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportApiCallRequest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.thingsboard.server.common.transport.queue; - -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class TransportApiCallRequest extends TransportApiCall implements TbQueueMsg { - public static final String REQUEST_ID_HEADER = "requestId"; - public static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - - private final UUID requestId; - private final Map headers; - private final TransportProtos.TransportApiRequestMsg msg; - - public TransportApiCallRequest(UUID requestId, String responseTopic, TransportProtos.TransportApiRequestMsg msg) { - this.requestId = requestId; - this.headers = new HashMap<>(); - this.headers.put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - this.headers.put(RESPONSE_TOPIC_HEADER, stringToBytes(responseTopic)); - this.msg = msg; - } - - @Override - public UUID getKey() { - return requestId; - } - - @Override - public Map getHeaders() { - return null; - } - - @Override - public byte[] getData() { - return msg.toByteArray(); - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java index d4ce11afec..d0bbafc615 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java @@ -1,19 +1,22 @@ package org.thingsboard.server.common.transport.queue; import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; public interface TransportQueueProvider { - TbQueueProducer getTransportApiCallRequestsProducer(); + TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate(); - TbQueueConsumer getTransportApiCallResponsesConsumer(); + TbQueueProducer> getRuleEngineMsgProducer(); - TbQueueProducer getRuleEngineMsgProducer(); + TbQueueProducer> getTbCoreMsgProducer(); - TbQueueProducer getTbCoreMsgProducer(); - - TbQueueConsumer getTransportNotificationsConsumer(); + TbQueueConsumer> getTransportNotificationsConsumer(); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java index 0aaa8cef68..1a66ea9601 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -29,6 +33,10 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.queue.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import java.util.Random; import java.util.UUID; @@ -54,6 +62,15 @@ public abstract class AbstractTransportService implements TransportService { @Autowired private TransportQueueProvider queueProvider; + + protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; + + protected TbQueueProducer> ruleEngineMsgProducer; + + protected TbQueueProducer> tbCoreMsgProducer; + + protected TbQueueConsumer> transportNotificationsConsumer; + protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; @@ -296,8 +313,8 @@ public abstract class AbstractTransportService implements TransportService { return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); } - protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { - return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString(); + protected UUID getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) { + return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } public void init() { @@ -309,6 +326,10 @@ public abstract class AbstractTransportService implements TransportService { this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); this.transportCallbackExecutor = Executors.newWorkStealingPool(20); this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); + ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); + transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); } public void destroy() { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java index 17031791d5..58685db764 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,16 +16,14 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; @@ -43,22 +41,18 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; import org.thingsboard.server.kafka.AsyncCallbackTemplate; -import org.thingsboard.server.kafka.TBKafkaAdmin; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.time.Duration; +import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -97,10 +91,6 @@ public class RemoteTransportService extends AbstractTransportService { @Autowired private TbNodeIdProvider nodeIdProvider; - private TbKafkaRequestTemplate transportApiTemplate; - private TBKafkaProducerTemplate ruleEngineProducer; - private TBKafkaConsumerTemplate mainConsumer; - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); private volatile boolean stopped = false; @@ -109,67 +99,67 @@ public class RemoteTransportService extends AbstractTransportService { public void init() { super.init(); - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiRequestsTopic); - requestBuilder.encoder(new TransportApiRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("transport-api-client"); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new TransportApiResponseDecoder()); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - transportApiTemplate = builder.build(); - transportApiTemplate.init(); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); - ruleEngineProducerBuilder.settings(kafkaSettings); - ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); - ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); - ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); - ruleEngineProducer = ruleEngineProducerBuilder.build(); - ruleEngineProducer.init(); - - String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); - - try { - TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); - CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - log.trace("Failed to create topic: {}", e.getMessage(), e); - } - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); - mainConsumerBuilder.settings(kafkaSettings); - mainConsumerBuilder.topic(notificationsTopicName); - mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - mainConsumerBuilder.groupId("transport"); - mainConsumerBuilder.autoCommit(true); - mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); - mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); - mainConsumer = mainConsumerBuilder.build(); - mainConsumer.subscribe(); +// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); +// requestBuilder.settings(kafkaSettings); +// requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); +// requestBuilder.defaultTopic(transportApiRequestsTopic); +// requestBuilder.encoder(new TransportApiRequestEncoder()); +// +// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); +// responseBuilder.settings(kafkaSettings); +// responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); +// responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); +// responseBuilder.groupId("transport-api-client"); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new TransportApiResponseDecoder()); +// +// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder +// builder = TbKafkaRequestTemplate.builder(); +// builder.requestTemplate(requestBuilder.build()); +// builder.responseTemplate(responseBuilder.build()); +// builder.maxPendingRequests(maxPendingRequests); +// builder.maxRequestTimeout(maxRequestsTimeout); +// builder.pollInterval(responsePollDuration); +// transportApiTemplate = builder.build(); +// transportApiTemplate.init(); +// +// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); +// ruleEngineProducerBuilder.settings(kafkaSettings); +// ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); +// ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); +// ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); +// ruleEngineProducer = ruleEngineProducerBuilder.build(); +// ruleEngineProducer.init(); +// +// String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); +// +// try { +// TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); +// CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); +// result.all().get(); +// } catch (Exception e) { +// log.trace("Failed to create topic: {}", e.getMessage(), e); +// } +// +// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); +// mainConsumerBuilder.settings(kafkaSettings); +// mainConsumerBuilder.topic(notificationsTopicName); +// mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); +// mainConsumerBuilder.groupId("transport"); +// mainConsumerBuilder.autoCommit(true); +// mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); +// mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); +// mainConsumer = mainConsumerBuilder.build(); +// mainConsumer.subscribe(); mainConsumerExecutor.execute(() -> { while (!stopped) { try { - ConsumerRecords records = mainConsumer.poll(Duration.ofMillis(notificationsPollDuration)); + List> records = transportNotificationsConsumer.poll(notificationsPollDuration); records.forEach(record -> { try { - ToTransportMsg toTransportMsg = mainConsumer.decode(record); + ToTransportMsg toTransportMsg = record.getValue(); if (toTransportMsg.hasToDeviceSessionMsg()) { processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); } @@ -193,12 +183,12 @@ public class RemoteTransportService extends AbstractTransportService { public void destroy() { super.destroy(); stopped = true; - if (transportApiTemplate != null) { - transportApiTemplate.stop(); - } - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } +// if (transportApiTemplate != null) { +// transportApiTemplate.stop(); +// } +// if (mainConsumer != null) { +// mainConsumer.unsubscribe(); +// } if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } @@ -207,25 +197,25 @@ public class RemoteTransportService extends AbstractTransportService { @Override public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(), - TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); } @Override public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(), - TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); } @Override public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); - AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(), - TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); } @Override @@ -322,18 +312,30 @@ public class RemoteTransportService extends AbstractTransportService { } private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineProducer.send(getRoutingKey(sessionInfo), toRuleEngineMsg, (metadata, exception) -> { - if (callback != null) { - if (exception == null) { - this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); - } else { - this.transportCallbackExecutor.submit(() -> { - callback.onError(exception); - }); - } - } - }); + ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + new TransportTbQueueCallback(callback) : null); + + } + + private class TransportTbQueueCallback implements TbQueueCallback { + private final TransportServiceCallback callback; + + private TransportTbQueueCallback(TransportServiceCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + RemoteTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onSuccess(null); + }); + } + + @Override + public void onFailure(Throwable t) { + RemoteTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onError(t); + }); + } } } From f8cc5b3cd2598b47207330e78a4bd23184f74fa6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 10 Mar 2020 17:47:16 +0200 Subject: [PATCH 004/196] Transport service refactoring --- .../transport/LocalTransportService.java | 227 ------------ .../transport/mqtt/MqttTransportHandler.java | 8 +- .../mqtt/session/GatewaySessionHandler.java | 6 +- ...vice.java => DefaultTransportService.java} | 247 ++++++++++--- .../service/RemoteTransportService.java | 341 ------------------ 5 files changed, 194 insertions(+), 635 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java rename common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/{AbstractTransportService.java => DefaultTransportService.java} (62%) delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java deleted file mode 100644 index 32ff77877d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportService.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.service.AbstractTransportService; -import org.thingsboard.server.dao.device.ClaimDevicesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 12.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "local") -public class LocalTransportService extends AbstractTransportService implements RuleEngineTransportService { - - @Autowired - private TransportApiService transportApiService; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - @Autowired - private ClaimDevicesService claimDevicesService; - - @PostConstruct - public void init() { - super.init(); - } - - @PreDestroy - public void destroy() { - super.destroy(); - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getValidateTokenResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - DonAsynchron.withCallback( - transportApiService.handle(TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()), - transportApiResponseMsg -> { - if (callback != null) { - callback.onSuccess(transportApiResponseMsg.getGetOrCreateDeviceResponseMsg()); - } - }, - getThrowableConsumer(callback), transportCallbackExecutor); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback); - } - - @Override - public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - TransportToDeviceActorMsg toDeviceActorMsg = TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setClaimDevice(msg).build(); - - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - callback.onSuccess(null); - } else { - TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB())); - DonAsynchron.withCallback(claimDevicesService.registerClaimingInfo(tenantId, deviceId, msg.getSecretKey(), msg.getDurationMs()), - callback::onSuccess, callback::onError); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - processToTransportMsg(msg); - if (onSuccess != null) { - onSuccess.run(); - } - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - if (callback != null) { - callback.onSuccess(null); - } - } - - private Consumer getThrowableConsumer(TransportServiceCallback callback) { - return e -> { - if (callback != null) { - callback.onError(e); - } - }; - } - -} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 84c7598d9b..0d96f94252 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; @@ -411,7 +411,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement ctx.close(); log.info("[{}] Client disconnected!", sessionId); if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); if (gatewaySessionHandler != null) { gatewaySessionHandler.onGatewayDisconnect(); @@ -486,7 +486,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); } } @@ -506,7 +506,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) .build(); - transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null); + transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); checkGatewaySession(); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index e0a944ed60..de5f82f324 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.service.AbstractTransportService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; @@ -121,7 +121,7 @@ public class GatewaySessionHandler { if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); } @@ -376,7 +376,7 @@ public class GatewaySessionHandler { private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) { transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); - transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + transportService.process(deviceSessionCtx.getSessionInfo(), DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java similarity index 62% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 1a66ea9601..dd7d9bd27a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -18,8 +18,11 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; @@ -37,16 +40,27 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.AsyncCallbackTemplate; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; import java.util.Random; import java.util.UUID; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * Created by ashvayka on 17.10.18. */ @Slf4j -public abstract class AbstractTransportService implements TransportService { +@Service +public class DefaultTransportService implements TransportService { @Value("${transport.rate_limits.enabled}") private boolean rateLimitEnabled; @@ -62,34 +76,134 @@ public abstract class AbstractTransportService implements TransportService { @Autowired private TransportQueueProvider queueProvider; + @Value("${kafka.notifications.poll_interval}") + private int notificationsPollDuration; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; - protected TbQueueProducer> ruleEngineMsgProducer; - protected TbQueueProducer> tbCoreMsgProducer; - protected TbQueueConsumer> transportNotificationsConsumer; protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; private ConcurrentMap sessions = new ConcurrentHashMap<>(); - //TODO: Implement cleanup of this maps. private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); + private volatile boolean stopped = false; + + @PostConstruct + public void init() { + if (rateLimitEnabled) { + //Just checking the configuration parameters + new TbRateLimits(perTenantLimitsConf); + new TbRateLimits(perDevicesLimitsConf); + } + this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); + this.transportCallbackExecutor = Executors.newWorkStealingPool(20); + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); + transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); + ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); + transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> records = transportNotificationsConsumer.poll(notificationsPollDuration); + records.forEach(record -> { + try { + ToTransportMsg toTransportMsg = record.getValue(); + if (toTransportMsg.hasToDeviceSessionMsg()) { + processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); + } + } catch (Throwable e) { + log.warn("Failed to process the notification.", e); + } + }); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + @PreDestroy + public void destroy() { + if (rateLimitEnabled) { + perTenantLimits.clear(); + perDeviceLimits.clear(); + } + stopped = true; + if (schedulerExecutor != null) { + schedulerExecutor.shutdownNow(); + } + if (transportCallbackExecutor != null) { + transportCallbackExecutor.shutdownNow(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + @Override public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } + @Override + public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { + log.trace("Processing msg: {}", msg); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), + response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); + } + + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { + if (log.isTraceEnabled()) { + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + } + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); + } + @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -97,7 +211,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostTelemetry(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -105,7 +223,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setPostAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -113,7 +235,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -122,7 +248,11 @@ public abstract class AbstractTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -131,7 +261,11 @@ public abstract class AbstractTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -139,7 +273,11 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @@ -147,14 +285,22 @@ public abstract class AbstractTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - doProcess(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToServerRPCCallRequest(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } } @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { - registerClaimingInfo(sessionInfo, msg, callback); + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build() + ).build(); + send(sessionInfo, toRuleEngineMsg, callback); } @Override @@ -162,24 +308,6 @@ public abstract class AbstractTransportService implements TransportService { reportActivityInternal(sessionInfo); } - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback); - - protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback); - - protected abstract void registerClaimingInfo(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback); - private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { UUID sessionId = toId(sessionInfo); SessionMetaData sessionMetaData = sessions.get(sessionId); @@ -317,37 +445,36 @@ public abstract class AbstractTransportService implements TransportService { return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } - public void init() { - if (rateLimitEnabled) { - //Just checking the configuration parameters - new TbRateLimits(perTenantLimitsConf); - new TbRateLimits(perDevicesLimitsConf); - } - this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); - this.transportCallbackExecutor = Executors.newWorkStealingPool(20); - this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); - ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); - tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); - transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { + return TransportProtos.SessionEventMsg.newBuilder() + .setSessionType(TransportProtos.SessionType.ASYNC) + .setEvent(event).build(); } - public void destroy() { - if (rateLimitEnabled) { - perTenantLimits.clear(); - perDeviceLimits.clear(); - } - if (schedulerExecutor != null) { - schedulerExecutor.shutdownNow(); + protected void send(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + new TransportTbQueueCallback(callback) : null); + } + + private class TransportTbQueueCallback implements TbQueueCallback { + private final TransportServiceCallback callback; + + private TransportTbQueueCallback(TransportServiceCallback callback) { + this.callback = callback; } - if (transportCallbackExecutor != null) { - transportCallbackExecutor.shutdownNow(); + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onSuccess(null); + }); } - } - public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { - return TransportProtos.SessionEventMsg.newBuilder() - .setSessionType(TransportProtos.SessionType.ASYNC) - .setEvent(event).build(); + @Override + public void onFailure(Throwable t) { + DefaultTransportService.this.transportCallbackExecutor.submit(() -> { + callback.onError(t); + }); + } } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java deleted file mode 100644 index 58685db764..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RemoteTransportService.java +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.transport.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.kafka.AsyncCallbackTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Created by ashvayka on 05.10.18. - */ -@ConditionalOnExpression("'${transport.type:null}'=='null'") -@Service -@Slf4j -public class RemoteTransportService extends AbstractTransportService { - - @Value("${kafka.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${kafka.notifications.topic}") - private String notificationsTopic; - @Value("${kafka.notifications.poll_interval}") - private int notificationsPollDuration; - @Value("${kafka.notifications.auto_commit_interval}") - private int notificationsAutoCommitInterval; - @Value("${kafka.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${kafka.transport_api.responses_topic}") - private String transportApiResponsesTopic; - @Value("${kafka.transport_api.max_pending_requests}") - private long maxPendingRequests; - @Value("${kafka.transport_api.max_requests_timeout}") - private long maxRequestsTimeout; - @Value("${kafka.transport_api.response_poll_interval}") - private int responsePollDuration; - @Value("${kafka.transport_api.response_auto_commit_interval}") - private int autoCommitInterval; - - @Autowired - private TbKafkaSettings kafkaSettings; - //We use this to get the node id. We should replace this with a component that provides the node id. - @Autowired - private TbNodeIdProvider nodeIdProvider; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("remote-transport-consumer")); - - private volatile boolean stopped = false; - - @PostConstruct - public void init() { - super.init(); - -// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); -// requestBuilder.settings(kafkaSettings); -// requestBuilder.clientId("producer-transport-api-request-" + nodeIdProvider.getNodeId()); -// requestBuilder.defaultTopic(transportApiRequestsTopic); -// requestBuilder.encoder(new TransportApiRequestEncoder()); -// -// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); -// responseBuilder.settings(kafkaSettings); -// responseBuilder.topic(transportApiResponsesTopic + "." + nodeIdProvider.getNodeId()); -// responseBuilder.clientId("transport-api-client-" + nodeIdProvider.getNodeId()); -// responseBuilder.groupId("transport-api-client"); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); -// -// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder -// builder = TbKafkaRequestTemplate.builder(); -// builder.requestTemplate(requestBuilder.build()); -// builder.responseTemplate(responseBuilder.build()); -// builder.maxPendingRequests(maxPendingRequests); -// builder.maxRequestTimeout(maxRequestsTimeout); -// builder.pollInterval(responsePollDuration); -// transportApiTemplate = builder.build(); -// transportApiTemplate.init(); -// -// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder ruleEngineProducerBuilder = TBKafkaProducerTemplate.builder(); -// ruleEngineProducerBuilder.settings(kafkaSettings); -// ruleEngineProducerBuilder.clientId("producer-rule-engine-request-" + nodeIdProvider.getNodeId()); -// ruleEngineProducerBuilder.defaultTopic(ruleEngineTopic); -// ruleEngineProducerBuilder.encoder(new ToRuleEngineMsgEncoder()); -// ruleEngineProducer = ruleEngineProducerBuilder.build(); -// ruleEngineProducer.init(); -// -// String notificationsTopicName = notificationsTopic + "." + nodeIdProvider.getNodeId(); -// -// try { -// TBKafkaAdmin admin = new TBKafkaAdmin(kafkaSettings); -// CreateTopicsResult result = admin.createTopic(new NewTopic(notificationsTopicName, 1, (short) 1)); -// result.all().get(); -// } catch (Exception e) { -// log.trace("Failed to create topic: {}", e.getMessage(), e); -// } -// -// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder mainConsumerBuilder = TBKafkaConsumerTemplate.builder(); -// mainConsumerBuilder.settings(kafkaSettings); -// mainConsumerBuilder.topic(notificationsTopicName); -// mainConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); -// mainConsumerBuilder.groupId("transport"); -// mainConsumerBuilder.autoCommit(true); -// mainConsumerBuilder.autoCommitIntervalMs(notificationsAutoCommitInterval); -// mainConsumerBuilder.decoder(new ToTransportMsgResponseDecoder()); -// mainConsumer = mainConsumerBuilder.build(); -// mainConsumer.subscribe(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> records = transportNotificationsConsumer.poll(notificationsPollDuration); - records.forEach(record -> { - try { - ToTransportMsg toTransportMsg = record.getValue(); - if (toTransportMsg.hasToDeviceSessionMsg()) { - processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @PreDestroy - public void destroy() { - super.destroy(); - stopped = true; -// if (transportApiTemplate != null) { -// transportApiTemplate.stop(); -// } -// if (mainConsumer != null) { -// mainConsumer.unsubscribe(); -// } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - @Override - public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback callback) { - log.trace("Processing msg: {}", msg); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()); - AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), - response -> callback.onSuccess(response.getValue().getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor); - } - - @Override - public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback) { - if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); - } - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostTelemetry(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToServerRPCCallRequest(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - @Override - protected void registerClaimingInfo(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); - } - - private void send(SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? - new TransportTbQueueCallback(callback) : null); - - } - - private class TransportTbQueueCallback implements TbQueueCallback { - private final TransportServiceCallback callback; - - private TransportTbQueueCallback(TransportServiceCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - RemoteTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); - } - - @Override - public void onFailure(Throwable t) { - RemoteTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onError(t); - }); - } - } -} From 4ad4fe11daec122d021a96def79b16d3197c460c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 11 Mar 2020 17:18:35 +0200 Subject: [PATCH 005/196] created in memory queue --- .../device/DeviceActorMessageProcessor.java | 4 +- .../service/script/RemoteJsInvokeService.java | 86 +++++++-------- .../script/RemoteJsRequestEncoder.java | 7 +- .../script/RemoteJsResponseDecoder.java | 10 +- .../transport/LocalTransportApiService.java | 25 ++--- .../RemoteRuleEngineTransportService.java | 56 +++++----- .../transport/RemoteTransportApiService.java | 28 +++-- .../transport/ToRuleEngineMsgDecoder.java | 6 +- .../transport/TransportApiService.java | 5 +- .../thingsboard/server/TbQueueConsumer.java | 2 + .../thingsboard/server/TbQueueMsgHeaders.java | 4 + .../common/DefaultTbQueueMsgHeaders.java | 7 +- .../server/common/TbProtoQueueMsg.java | 4 +- .../server/kafka/KafkaTbQueueMsg.java | 39 +++++++ .../server/kafka/KafkaTbQueueMsgMetadata.java | 12 ++ .../server/kafka/TBKafkaAdmin.java | 9 +- .../server/kafka/TBKafkaConsumerTemplate.java | 48 +++++++- .../server/kafka/TBKafkaProducerTemplate.java | 104 +++++++++++------- .../server/kafka/TbKafkaDecoder.java | 4 +- .../server/memory/InMemoryStorage.java | 45 ++++++++ .../memory/InMemoryTbQueueConsumer.java | 42 +++++++ .../memory/InMemoryTbQueueProducer.java | 43 ++++++++ .../service/DefaultTransportService.java | 2 +- .../ToTransportMsgResponseDecoder.java | 6 +- .../service/TransportApiResponseDecoder.java | 6 +- .../server/dao/entity/BaseEntityService.java | 13 ++- .../dao/relation/BaseRelationService.java | 38 ++++--- .../timescale/TimescaleTimeseriesDao.java | 3 +- .../CassandraBaseTimeseriesDao.java | 5 +- 29 files changed, 481 insertions(+), 182 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index acf1a3161c..2752b53090 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -20,9 +20,9 @@ import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -292,7 +292,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .build(); sendToTransport(responseMsg, sessionInfo); } - }); + }, MoreExecutors.directExecutor()); } private ListenableFuture>> getAttributesKvEntries(GetAttributeRequestMsg request) { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 45d773f387..d39465ff12 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -25,11 +25,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaRequestTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.Nullable; @@ -40,7 +39,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) @@ -99,42 +97,43 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - private TbKafkaRequestTemplate kafkaTemplate; + private DefaultTbQueueRequestTemplate, TbProtoQueueMsg> defaultTemplate; private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(requestTopic); - requestBuilder.encoder(new RemoteJsRequestEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.autoCommitIntervalMs(autoCommitInterval); - responseBuilder.decoder(new RemoteJsResponseDecoder()); - responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); - - TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder - builder = TbKafkaRequestTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); - builder.maxPendingRequests(maxPendingRequests); - builder.maxRequestTimeout(maxRequestsTimeout); - builder.pollInterval(responsePollDuration); - kafkaTemplate = builder.build(); - kafkaTemplate.init(); +// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); +// requestBuilder.settings(kafkaSettings); +// requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); +// requestBuilder.defaultTopic(requestTopic); +// requestBuilder.encoder(new RemoteJsRequestEncoder()); + TbQueueProducer> producer; + +// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); +// responseBuilder.settings(kafkaSettings); +// responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); +// responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); +// responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new RemoteJsResponseDecoder()); +// responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); +// +// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder +// builder = TbKafkaRequestTemplate.builder(); +// builder.requestTemplate(requestBuilder.build()); +// builder.responseTemplate(responseBuilder.build()); +// builder.maxPendingRequests(maxPendingRequests); +// builder.maxRequestTimeout(maxRequestsTimeout); +// builder.pollInterval(responsePollDuration); +// defaultTemplate = builder.build(); +// defaultTemplate.init(); } @PreDestroy public void destroy() { - if (kafkaTemplate != null) { - kafkaTemplate.stop(); + if (defaultTemplate != null) { + defaultTemplate.stop(); } } @@ -151,11 +150,12 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaEvalMsgs.incrementAndGet(); } @@ -168,7 +168,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }); return Futures.transform(future, response -> { - JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse(); + JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); if (compilationResult.getSuccess()) { scriptIdToNameMap.put(scriptId, functionName); @@ -202,11 +202,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); + ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback>() { @Override - public void onSuccess(@Nullable JsInvokeProtos.RemoteJsResponse result) { + public void onSuccess(@Nullable TbProtoQueueMsg result) { kafkaInvokeMsgs.incrementAndGet(); } @@ -219,7 +219,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } }); return Futures.transform(future, response -> { - JsInvokeProtos.JsInvokeResponse invokeResult = response.getInvokeResponse(); + JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse(); if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { @@ -240,8 +240,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture future = kafkaTemplate.post(UUID.randomUUID().toString(), jsRequestWrapper); - JsInvokeProtos.RemoteJsResponse response = future.get(); + ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB()); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index d07a2490ba..1b33fd85cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.kafka.TbKafkaEncoder; @@ -25,11 +26,11 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsRequestEncoder implements TbKafkaEncoder { +public class RemoteJsRequestEncoder implements TbKafkaEncoder> { @Override - public byte[] encode(JsInvokeProtos.RemoteJsRequest value) { + public byte[] encode(TbProtoQueueMsg value) { try { - return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + return JsonFormat.printer().print(value.getValue()).getBytes(StandardCharsets.UTF_8); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index 7a3876fb91..ef15d5c877 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -25,12 +27,12 @@ import java.nio.charset.StandardCharsets; /** * Created by ashvayka on 25.09.18. */ -public class RemoteJsResponseDecoder implements TbKafkaDecoder { +public class RemoteJsResponseDecoder implements TbKafkaDecoder> { @Override - public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException { + public TbProtoQueueMsg decode(TbQueueMsg msg) throws IOException { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder); - return builder.build(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java index 37ff8ed93d..f6434036fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java @@ -21,10 +21,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -42,19 +41,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaResponseTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.locks.ReentrantLock; /** @@ -84,17 +74,18 @@ public class LocalTransportApiService implements TransportApiService { private ReentrantLock deviceCreationLock = new ReentrantLock(); @Override - public ListenableFuture handle(TransportApiRequestMsg transportApiRequestMsg) { + public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { + TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); + return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); + return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); + return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } - return getEmptyTransportApiResponseFuture(); + return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders())); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -145,7 +136,7 @@ public class LocalTransportApiService implements TransportApiService { try { ValidateDeviceCredentialsResponseMsg.Builder builder = ValidateDeviceCredentialsResponseMsg.newBuilder(); builder.setDeviceInfo(getDeviceInfoProto(device)); - if(!StringUtils.isEmpty(credentials.getCredentialsValue())){ + if (!StringUtils.isEmpty(credentials.getCredentialsValue())) { builder.setCredentialsBody(credentials.getCredentialsValue()); } return TransportApiResponseMsg.newBuilder() diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java index 7aa438c64f..83353cea4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -22,9 +22,6 @@ import io.github.bucket4j.Bucket4j; import io.github.bucket4j.local.LocalBucket; import io.github.bucket4j.local.LocalBucketBuilder; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -33,15 +30,17 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; @@ -51,6 +50,7 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; @@ -84,9 +84,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Value("${transport.remote.rule_engine.stats.enabled:false}") private boolean statsEnabled; - @Autowired - private TbKafkaSettings kafkaSettings; - +// @Autowired +// private TbKafkaSettings kafkaSettings; +// @Autowired private TbNodeIdProvider nodeIdProvider; @@ -101,8 +101,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Autowired private DataDecodingEncodingService encodingService; - private TBKafkaConsumerTemplate ruleEngineConsumer; - private TBKafkaProducerTemplate notificationsProducer; + private TbQueueConsumer> ruleEngineConsumer; + + private TbQueueProducer> notificationsProducer; private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); @@ -146,8 +147,8 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ mainConsumerExecutor.execute(() -> { while (!stopped) { try { - ConsumerRecords records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration)); - int recordsCount = records.count(); + List> msgs = ruleEngineConsumer.poll(pollDuration); + int recordsCount = msgs.size(); if (recordsCount > 0) { while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); @@ -155,9 +156,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ } log.trace("Processing {} records", recordsCount); } - records.forEach(record -> { + msgs.forEach(msg -> { try { - ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); if (toRuleEngineMsg.hasToDeviceActorMsg()) { forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); @@ -196,7 +197,8 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); + notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { @@ -225,7 +227,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ } } - private static class QueueCallbackAdaptor implements Callback { + private static class QueueCallbackAdaptor implements TbQueueCallback { private final Runnable onSuccess; private final Consumer onFailure; @@ -235,17 +237,17 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ } @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - if (onSuccess != null) { - onSuccess.run(); - } - } else { - if (onFailure != null) { - onFailure.accept(exception); - } + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); } } - } + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 0c9efc7d1c..e354da7daa 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -22,9 +22,16 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.*; +import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -49,9 +56,9 @@ public class RemoteTransportApiService { @Value("${transport.remote.transport_api.request_auto_commit_interval}") private int autoCommitInterval; - @Autowired - private TbKafkaSettings kafkaSettings; - +// @Autowired +// private TbKafkaSettings kafkaSettings; +// @Autowired private TbNodeIdProvider nodeIdProvider; @@ -60,7 +67,7 @@ public class RemoteTransportApiService { private ExecutorService transportCallbackExecutor; - private TbKafkaResponseTemplate transportApiTemplate; + private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; @PostConstruct public void init() { @@ -79,11 +86,14 @@ public class RemoteTransportApiService { requestBuilder.autoCommit(true); requestBuilder.autoCommitIntervalMs(autoCommitInterval); requestBuilder.decoder(new TransportApiRequestDecoder()); + TbQueueProducer> producer = null; + TbQueueConsumer> consumer = null; + - TbKafkaResponseTemplate.TbKafkaResponseTemplateBuilder - builder = TbKafkaResponseTemplate.builder(); - builder.requestTemplate(requestBuilder.build()); - builder.responseTemplate(responseBuilder.build()); + DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); + builder.requestTemplate(consumer); + builder.responseTemplate(producer); builder.maxPendingRequests(maxPendingRequests); builder.requestTimeout(requestTimeout); builder.pollInterval(responsePollDuration); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java index 9f08463f91..bdebacbd13 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.transport; +import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { + @Override - public ToRuleEngineMsg decode(byte[] data) throws IOException { - return ToRuleEngineMsg.parseFrom(data); + public ToRuleEngineMsg decode(TbQueueMsg msg) throws IOException { + return ToRuleEngineMsg.parseFrom(msg.getData()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index 2964934313..7d572d08ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,11 +15,12 @@ */ package org.thingsboard.server.service.transport; +import org.thingsboard.server.TbQueueHandler; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.kafka.TbKafkaHandler; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbKafkaHandler { +public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index 181d50bdd1..ca51eb4ddb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -8,6 +8,8 @@ public interface TbQueueConsumer { void subscribe(); + void unsubscribe(); + List poll(long durationInMillis); void commit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java index e1a5b4861d..9dad87588d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -1,8 +1,12 @@ package org.thingsboard.server; +import java.util.Map; + public interface TbQueueMsgHeaders { byte[] put(String key, byte[] value); byte[] get(String key); + + Map getData(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java index 4ef33a489d..e547a6c070 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -7,7 +7,7 @@ import java.util.Map; public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { - private final Map data = new HashMap<>(); + protected final Map data = new HashMap<>(); @Override public byte[] put(String key, byte[] value) { @@ -18,4 +18,9 @@ public class DefaultTbQueueMsgHeaders implements TbQueueMsgHeaders { public byte[] get(String key) { return data.get(key); } + + @Override + public Map getData() { + return data; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java index f1120b876f..c072dc4d67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -11,13 +11,13 @@ public class TbProtoQueueMsg i private final UUID key; private final T value; - private final DefaultTbQueueMsgHeaders headers; + private final TbQueueMsgHeaders headers; public TbProtoQueueMsg(UUID key, T value) { this(key, value, new DefaultTbQueueMsgHeaders()); } - public TbProtoQueueMsg(UUID key, T value, DefaultTbQueueMsgHeaders headers) { + public TbProtoQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { this.key = key; this.value = value; this.headers = headers; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java new file mode 100644 index 0000000000..e44a93a70d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java @@ -0,0 +1,39 @@ +package org.thingsboard.server.kafka; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgHeaders; +import org.thingsboard.server.common.DefaultTbQueueMsgHeaders; + +import java.util.UUID; + +public class KafkaTbQueueMsg implements TbQueueMsg { + private final UUID key; + private final TbQueueMsgHeaders headers; + private final byte[] data; + + public KafkaTbQueueMsg(ConsumerRecord record) { + this.key = UUID.fromString(record.key()); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + record.headers().forEach(header -> { + headers.put(header.key(), header.value()); + }); + this.headers = headers; + this.data = record.value(); + } + + @Override + public UUID getKey() { + return key; + } + + @Override + public TbQueueMsgHeaders getHeaders() { + return headers; + } + + @Override + public byte[] getData() { + return data; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java new file mode 100644 index 0000000000..4698c226d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java @@ -0,0 +1,12 @@ +package org.thingsboard.server.kafka; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.thingsboard.server.TbQueueMsgMetadata; + +@Data +@AllArgsConstructor +public class KafkaTbQueueMsgMetadata implements TbQueueMsgMetadata { + private RecordMetadata metadata; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java index 0fe86e030b..9c6d911d33 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.kafka; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.KafkaFuture; +import org.thingsboard.server.TbQueueAdmin; import java.time.Duration; import java.util.Collections; @@ -30,7 +32,7 @@ import java.util.concurrent.TimeoutException; /** * Created by ashvayka on 24.09.18. */ -public class TBKafkaAdmin { +public class TBKafkaAdmin implements TbQueueAdmin { AdminClient client; @@ -66,4 +68,9 @@ public class TBKafkaAdmin { } } + @Override + public ListenableFuture createTopicIfNotExists(String topic) { + + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 73f8cc16c7..6f0ce7b55c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -17,21 +17,27 @@ package org.thingsboard.server.kafka; import lombok.Builder; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Properties; import java.util.UUID; /** * Created by ashvayka on 24.09.18. */ -public class TBKafkaConsumerTemplate { +@Slf4j +public class TBKafkaConsumerTemplate implements TbQueueConsumer { private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; @@ -66,23 +72,55 @@ public class TBKafkaConsumerTemplate { this.topic = topic; } + @Override public void subscribe() { consumer.subscribe(Collections.singletonList(topic)); } + @Override + public List poll(long durationInMillis) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } + + return Collections.emptyList(); + } + + @Override + public void commit() { + consumer.commitAsync(); + } + + @Override public void unsubscribe() { consumer.unsubscribe(); } - public ConsumerRecords poll(Duration duration) { - return consumer.poll(duration); - } +// public void subscribe() { +// consumer.subscribe(Collections.singletonList(topic)); +// } +// + +// +// public ConsumerRecords poll(Duration duration) { +// return consumer.poll(duration); +// } public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(record.value()); + return decoder.decode(new KafkaTbQueueMsg(record)); } public UUID extractRequestId(T value) { return requestIdExtractor.extractRequestId(value); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 8722c1d64e..938b278207 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -15,42 +15,45 @@ */ package org.thingsboard.server.kafka; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.JdkFutureAdapters; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; import java.util.List; import java.util.Properties; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaProducerTemplate { +public class TBKafkaProducerTemplate implements TbQueueProducer { private final KafkaProducer producer; - private final TbKafkaEncoder encoder; private final TbKafkaPartitioner partitioner; + private ConcurrentMap> partitionInfoMap; + @Getter private final String defaultTopic; @@ -58,8 +61,7 @@ public class TBKafkaProducerTemplate { private final TbKafkaSettings settings; @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder encoder, - TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { + private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { Properties props = settings.toProps(); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); @@ -68,7 +70,6 @@ public class TBKafkaProducerTemplate { } this.settings = settings; this.producer = new KafkaProducer<>(props); - this.encoder = encoder; this.partitioner = partitioner; this.defaultTopic = defaultTopic; } @@ -89,43 +90,68 @@ public class TBKafkaProducerTemplate { } } - public Future send(String key, T value, Callback callback) { - return send(key, value, null, callback); - } - - public Future send(String key, T value, Iterable
headers, Callback callback) { - return send(key, value, null, headers, callback); + @Override + public ListenableFuture send(T msg, TbQueueCallback callback) { + return send(defaultTopic, msg, callback); } - public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { - if (!StringUtils.isEmpty(this.defaultTopic)) { - return send(this.defaultTopic, key, value, timestamp, headers, callback); - } else { - throw new RuntimeException("Failed to send message! Default topic is not specified!"); - } - } + @Override + public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + String key = msg.getKey().toString(); + byte[] data = msg.getData(); + ProducerRecord record; + Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { - return send(topic, key, value, null, headers, callback); - } + Integer partition = getPartition(topic, msg); + record = new ProducerRecord<>(topic, partition, key, data, headers); + Future result = producer.send(record, (metadata, exception) -> { + if (exception == null) { + callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + } else { + callback.onFailure(exception); + } + }); - public Future send(String topic, String key, T value, Callback callback) { - return send(topic, key, value, null, null, callback); + return Futures.transform(JdkFutureAdapters.listenInPoolThread(result), metadata -> new KafkaTbQueueMsgMetadata(metadata)); } - public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { - byte[] data = encoder.encode(value); - ProducerRecord record; - Integer partition = getPartition(topic, key, value, data); - record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); - return producer.send(record, callback); - } +// public Future send(String key, T value, Callback callback) { +// return send(key, value, null, callback); +// } +// +// public Future send(String key, T value, Iterable
headers, Callback callback) { +// return send(key, value, null, headers, callback); +// } +// +// public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { +// if (!StringUtils.isEmpty(this.defaultTopic)) { +// return send(this.defaultTopic, key, value, timestamp, headers, callback); +// } else { +// throw new RuntimeException("Failed to send message! Default topic is not specified!"); +// } +// } +// +// public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { +// return send(topic, key, value, null, headers, callback); +// } +// +// public Future send(String topic, String key, T value, Callback callback) { +// return send(topic, key, value, null, null, callback); +// } +// +// public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { +// byte[] data = encoder.encode(value); +// ProducerRecord record; +// Integer partition = getPartition(topic, key, value, data); +// record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); +// return producer.send(record, callback); +// } - private Integer getPartition(String topic, String key, T value, byte[] data) { + private Integer getPartition(String topic, T value) { if (partitioner == null) { return null; } else { - return partitioner.partition(topic, key, value, data, partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); + return partitioner.partition(topic, value.getKey().toString(), value, value.getData(), partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java index ab196d5863..564d905675 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.kafka; +import org.thingsboard.server.TbQueueMsg; + import java.io.IOException; /** @@ -22,6 +24,6 @@ import java.io.IOException; */ public interface TbKafkaDecoder { - T decode(byte[] data) throws IOException; + T decode(TbQueueMsg msg) throws IOException; } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java new file mode 100644 index 0000000000..abf30667b5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -0,0 +1,45 @@ +package org.thingsboard.server.memory; + +import org.thingsboard.server.TbQueueMsg; + +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; + +public final class InMemoryStorage { + private static InMemoryStorage instance; + private final Map> storage; + + private InMemoryStorage() { + storage = new ConcurrentHashMap<>(); + } + + public static InMemoryStorage getInstance() { + if (instance == null) { + synchronized (InMemoryStorage.class) { + if (instance == null) { + instance = new InMemoryStorage(); + } + } + } + return instance; + } + + public boolean put(String topic, TbQueueMsg msg) { + return storage.computeIfAbsent(topic, (t) -> new LinkedList<>()).add(msg); + } + + public TbQueueMsg get(String topic) { + if (storage.containsKey(topic)) { + return storage.get(topic).peek(); + } + return null; + } + + public void commit(String topic) { + if (storage.containsKey(topic)) { + storage.get(topic).remove(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java new file mode 100644 index 0000000000..a0a35eefb8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -0,0 +1,42 @@ +package org.thingsboard.server.memory; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueMsg; + +import java.util.Collections; +import java.util.List; + +public class InMemoryTbQueueConsumer implements TbQueueConsumer { + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + + public InMemoryTbQueueConsumer(String topic) { + this.topic = topic; + } + + private final String topic; + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + + } + + @Override + public void unsubscribe() { + + } + + @Override + public List poll(long durationInMillis) { + return Collections.singletonList((T)storage.get(topic)); + } + + @Override + public void commit() { + storage.commit(topic); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java new file mode 100644 index 0000000000..3b53ddf316 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -0,0 +1,43 @@ +package org.thingsboard.server.memory; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; + +public class InMemoryTbQueueProducer implements TbQueueProducer { + + private final InMemoryStorage storage = InMemoryStorage.getInstance(); + + private String defaultTopic; + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public ListenableFuture send(T msg, TbQueueCallback callback) { + return send(defaultTopic, msg, callback); + } + + @Override + public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + boolean result = storage.put(topic, msg); + if (result) { + callback.onSuccess(null); + return Futures.immediateCheckedFuture(null); + } else { + Exception e = new RuntimeException("Failure add msg to InMemoryQueue"); + callback.onFailure(e); + return Futures.immediateFailedFuture(e); + } + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index dd7d9bd27a..921003dbd7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -40,7 +40,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.AsyncCallbackTemplate; +import org.thingsboard.server.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index c07e406d2e..31a32d8368 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class ToTransportMsgResponseDecoder implements TbKafkaDecoder { + @Override - public ToTransportMsg decode(byte[] data) throws IOException { - return ToTransportMsg.parseFrom(data); + public ToTransportMsg decode(TbQueueMsg msg) throws IOException { + return ToTransportMsg.parseFrom(msg.getData()); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index a62e696500..89d61fba56 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.transport.service; +import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.kafka.TbKafkaDecoder; @@ -24,8 +25,9 @@ import java.io.IOException; * Created by ashvayka on 05.10.18. */ public class TransportApiResponseDecoder implements TbKafkaDecoder { + @Override - public TransportApiResponseMsg decode(byte[] data) throws IOException { - return TransportApiResponseMsg.parseFrom(data); + public TransportApiResponseMsg decode(TbQueueMsg msg) throws IOException { + return TransportApiResponseMsg.parseFrom(msg.getData()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index c49fcc3728..5867e505b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -18,12 +18,21 @@ package org.thingsboard.server.dao.entity; import com.google.common.base.Function; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.alarm.AlarmId; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; @@ -109,7 +118,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe default: throw new IllegalStateException("Not Implemented!"); } - entityName = Futures.transform(hasName, (Function) hasName1 -> hasName1 != null ? hasName1.getName() : null ); + entityName = Futures.transform(hasName, (Function) hasName1 -> hasName1 != null ? hasName1.getName() : null, MoreExecutors.directExecutor()); return entityName; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 68b7fce6a8..22b2543489 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -16,7 +16,10 @@ package org.thingsboard.server.dao.relation; import com.google.common.base.Function; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.Cache; @@ -206,17 +209,20 @@ public class BaseRelationService implements RelationService { relations -> { List> results = deleteRelationGroupsAsync(tenantId, relations, cache, true); return Futures.allAsList(results); - }); + }, MoreExecutors.directExecutor()); ListenableFuture> outboundDeletions = Futures.transformAsync(outboundRelations, relations -> { List> results = deleteRelationGroupsAsync(tenantId, relations, cache, false); return Futures.allAsList(results); - }); + }, MoreExecutors.directExecutor()); ListenableFuture>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions); - return Futures.transform(Futures.transformAsync(deletionsFuture, (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId)), result -> null); + return Futures.transform(Futures.transformAsync(deletionsFuture, + (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId), + MoreExecutors.directExecutor()), + result -> null, MoreExecutors.directExecutor()); } private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, Cache cache, boolean deleteFromDb) { @@ -306,9 +312,11 @@ public class BaseRelationService implements RelationService { public void onSuccess(@Nullable List result) { cache.putIfAbsent(fromAndTypeGroup, result); } + @Override - public void onFailure(Throwable t) {} - }); + public void onFailure(Throwable t) { + } + }, MoreExecutors.directExecutor()); return relationsFuture; } } @@ -328,7 +336,7 @@ public class BaseRelationService implements RelationService { EntityRelationInfo::setToName)) ); return Futures.successfulAsList(futures); - }); + }, MoreExecutors.directExecutor()); } @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #relationType, #typeGroup, 'FROM'}") @@ -385,9 +393,11 @@ public class BaseRelationService implements RelationService { public void onSuccess(@Nullable List result) { cache.putIfAbsent(toAndTypeGroup, result); } + @Override - public void onFailure(Throwable t) {} - }); + public void onFailure(Throwable t) { + } + }, MoreExecutors.directExecutor()); return relationsFuture; } } @@ -407,7 +417,7 @@ public class BaseRelationService implements RelationService { EntityRelationInfo::setFromName)) ); return Futures.successfulAsList(futures); - }); + }, MoreExecutors.directExecutor()); } private ListenableFuture fetchRelationInfoAsync(TenantId tenantId, EntityRelation relation, @@ -418,7 +428,7 @@ public class BaseRelationService implements RelationService { EntityRelationInfo entityRelationInfo1 = new EntityRelationInfo(relation); entityNameSetter.accept(entityRelationInfo1, entityName1); return entityRelationInfo1; - }); + }, MoreExecutors.directExecutor()); } @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#to, #relationType, #typeGroup, 'TO'}") @@ -466,7 +476,7 @@ public class BaseRelationService implements RelationService { } } return relations; - }); + }, MoreExecutors.directExecutor()); } catch (Exception e) { log.warn("Failed to query relations: [{}]", query, e); throw new RuntimeException(e); @@ -493,7 +503,7 @@ public class BaseRelationService implements RelationService { })) ); return Futures.successfulAsList(futures); - }); + }, MoreExecutors.directExecutor()); } protected void validate(EntityRelation relation) { @@ -600,7 +610,7 @@ public class BaseRelationService implements RelationService { } //TODO: try to remove this blocking operation List> relations = Futures.successfulAsList(futures).get(); - if (fetchLastLevelOnly && lvl > 0){ + if (fetchLastLevelOnly && lvl > 0) { children.clear(); } relations.forEach(r -> r.forEach(children::add)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 176bc712e8..4ca53a337b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sqlts.timescale; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -143,7 +144,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } else { return Collections.emptyList(); } - }); + }, MoreExecutors.directExecutor()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 8fc8b4ab8a..b96462e350 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -330,7 +331,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem stmt.setInt(6, (int) ttl); } futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null); + return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); } private void processSetNullValues(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl, List> futures, long partition, DataType type) { @@ -545,7 +546,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem public void onFailure(Throwable t) { log.warn("[{}] Failed to process remove of the latest value", entityId, t); } - }); + }, MoreExecutors.directExecutor()); return resultFuture; } From 6e31c0ab157172897bcacd6e1b22b049302400d8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 11 Mar 2020 20:09:41 +0200 Subject: [PATCH 006/196] Refactoring of configuration properties --- .../src/main/resources/thingsboard.yml | 75 +++++++++---------- .../memory/InMemoryTbQueueProducer.java | 4 +- .../queue/KafkaTransportQueueProvider.java | 4 + .../service/DefaultTransportService.java | 38 ++++++---- .../src/main/resources/tb-mqtt-transport.yml | 34 +++++---- 5 files changed, 83 insertions(+), 72 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 820fe14be1..0cd5be8fc8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -428,28 +428,6 @@ state: defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - js: evaluator: "${JS_EVALUATOR:local}" # local/remote # Built-in JVM JavaScript environment properties @@ -489,25 +467,6 @@ js: transport: type: "${TRANSPORT_TYPE:local}" # local or remote - remote: - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - request_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:1000}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}" - poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}" - max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}" - max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}" - stats: - enabled: "${TB_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -573,3 +532,37 @@ swagger: title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" + + +queue: + type: "${TB_QUEUE_TYPE:kafka}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + rule_engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + notifications: + topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + +rule_engine: + type: "${RULE_ENGINE_TYPE:local}" # local or remote \ No newline at end of file diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 3b53ddf316..32b9399164 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -2,16 +2,18 @@ package org.thingsboard.server.memory; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; +@Data public class InMemoryTbQueueProducer implements TbQueueProducer { private final InMemoryStorage storage = InMemoryStorage.getInstance(); - private String defaultTopic; + private final String defaultTopic; @Override public void init() { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java index 2aa6f90b7b..e16c7fd93c 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java @@ -1,5 +1,7 @@ package org.thingsboard.server.common.transport.queue; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; @@ -8,6 +10,8 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; @Component +@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") +@Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 921003dbd7..9ccb5369f8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -192,7 +192,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscriptionInfo(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } @Override @@ -203,7 +203,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSessionEvent(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -215,7 +215,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setPostTelemetry(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); } } @@ -227,7 +227,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setPostAttributes(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); } } @@ -239,7 +239,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setGetAttributes(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -252,7 +252,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscribeToAttributes(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -265,7 +265,7 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscribeToRPC(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } @@ -277,10 +277,11 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setToDeviceRPCCallResponse(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); } } + //TODO 2.5: Need to handle timeouts on the transport level and not on the Device Actor Level. @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { @@ -289,18 +290,20 @@ public class DefaultTransportService implements TransportService { TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setToServerRPCCallRequest(msg).build() ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); } } @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - send(sessionInfo, toRuleEngineMsg, callback); + if (checkLimits(sessionInfo, msg, callback)) { + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( + TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build() + ).build(); + sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + } } @Override @@ -451,7 +454,12 @@ public class DefaultTransportService implements TransportService { .setEvent(event).build(); } - protected void send(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + new TransportTbQueueCallback(callback) : null); + } + + protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? new TransportTbQueueCallback(callback) : null); } diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 47dfe57c85..741adc6795 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -61,24 +61,28 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - queue: - type: kafka + +queue: + type: "${TB_QUEUE_TYPE:kafka}" + kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" - transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + rule_engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + notifications: + topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" From 52814d2bfcb7e11feb44ccf26a81fc96bd337637 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 12 Mar 2020 09:36:48 +0200 Subject: [PATCH 007/196] Refactoring of Transport Communication to Queues --- ...e.java => DefaultTransportApiService.java} | 2 +- .../transport/RemoteTransportApiService.java | 57 ++++------- .../transport/TransportApiService.java | 5 +- .../src/main/resources/thingsboard.yml | 1 + common/queue/pom.xml | 9 ++ .../provider/KafkaTbCoreQueueProvider.java | 42 ++++++++ .../provider/KafkaTransportQueueProvider.java | 39 ++++++++ .../server/provider/TbCoreQueueProvider.java | 63 ++++++++++++ .../provider}/TransportQueueProvider.java | 5 +- .../src/main/proto/transport.proto | 72 +++++++++----- common/transport/transport-api/pom.xml | 13 --- .../queue/KafkaTransportQueueProvider.java | 35 ------- .../service/DefaultTransportService.java | 98 +++++++------------ 13 files changed, 260 insertions(+), 181 deletions(-) rename application/src/main/java/org/thingsboard/server/service/transport/{LocalTransportApiService.java => DefaultTransportApiService.java} (99%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue => queue/src/main/java/org/thingsboard/server/provider}/TransportQueueProvider.java (82%) rename common/{transport/transport-api => queue}/src/main/proto/transport.proto (75%) delete mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index f6434036fd..51597877b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/LocalTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -52,7 +52,7 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -public class LocalTransportApiService implements TransportApiService { +public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index e354da7daa..644d4191b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,6 +32,7 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.provider.TbCoreQueueProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -45,50 +46,32 @@ import java.util.concurrent.*; @ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { - @Value("${transport.remote.transport_api.requests_topic}") - private String transportApiRequestsTopic; - @Value("${transport.remote.transport_api.max_pending_requests}") + private final TbCoreQueueProvider tbCoreQueueProvider; + private final TransportApiService transportApiService; + + @Value("${queue.transport_api.max_pending_requests:10000}") private int maxPendingRequests; - @Value("${transport.remote.transport_api.request_timeout}") + @Value("${queue.transport_api.max_requests_timeout:10000}") private long requestTimeout; - @Value("${transport.remote.transport_api.request_poll_interval}") + @Value("${queue.transport_api.request_poll_interval:25}") private int responsePollDuration; - @Value("${transport.remote.transport_api.request_auto_commit_interval}") - private int autoCommitInterval; - -// @Autowired -// private TbKafkaSettings kafkaSettings; -// - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TransportApiService transportApiService; + @Value("${queue.transport_api.max_callback_threads:100}") + private int maxCallbackThreads; private ExecutorService transportCallbackExecutor; + private TbQueueResponseTemplate, + TbProtoQueueMsg> transportApiTemplate; - private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; + public RemoteTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { + this.tbCoreQueueProvider = tbCoreQueueProvider; + this.transportApiService = transportApiService; + } @PostConstruct public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(100); - - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder responseBuilder = TBKafkaProducerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.clientId("producer-transport-api-response-" + nodeIdProvider.getNodeId()); - responseBuilder.encoder(new TransportApiResponseEncoder()); - - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder requestBuilder = TBKafkaConsumerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.topic(transportApiRequestsTopic); - requestBuilder.clientId(nodeIdProvider.getNodeId()); - requestBuilder.groupId("tb-node"); - requestBuilder.autoCommit(true); - requestBuilder.autoCommitIntervalMs(autoCommitInterval); - requestBuilder.decoder(new TransportApiRequestDecoder()); - TbQueueProducer> producer = null; - TbQueueConsumer> consumer = null; - + this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); + TbQueueProducer> producer = tbCoreQueueProvider.getTransportApiResponseProducer(); + TbQueueConsumer> consumer = tbCoreQueueProvider.getTransportApiRequestConsumer(); DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); @@ -105,7 +88,7 @@ public class RemoteTransportApiService { @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(); + transportApiTemplate.init(transportApiService); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index 7d572d08ed..eab2e07b0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -17,10 +17,11 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.TbQueueHandler; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; /** * Created by ashvayka on 05.10.18. */ -public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { +public interface TransportApiService extends TbQueueHandler, TbProtoQueueMsg> { } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0cd5be8fc8..dc479191e7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -548,6 +548,7 @@ queue: responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 952b3c540d..fd26358cc6 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -100,4 +100,13 @@ + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java new file mode 100644 index 0000000000..eac81b80c3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -0,0 +1,42 @@ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +@Component +@ConditionalOnExpression("'${transport.type:null}'=='local' || '${transport.type:null}'=='remote'") +public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider{ + @Override + public TbQueueProducer> getTransportMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return null; + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return null; + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java new file mode 100644 index 0000000000..6686b3704f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -0,0 +1,39 @@ +package org.thingsboard.server.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +@Component +@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") +@Slf4j +public class KafkaTransportQueueProvider implements TransportQueueProvider { + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + return null; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return null; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return null; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java new file mode 100644 index 0000000000..b7e210f0f6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -0,0 +1,63 @@ +package org.thingsboard.server.provider; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbCoreQueueProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToCoreMsgConsumer(); + + /** + * Used to consume Transport API Calls + * + * @return + */ + TbQueueConsumer> getTransportApiRequestConsumer(); + + /** + * Used to push replies to Transport API Calls + * + * @return + */ + TbQueueProducer> getTransportApiResponseProducer(); + + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java similarity index 82% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java index d0bbafc615..14d8efd534 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java @@ -1,9 +1,10 @@ -package org.thingsboard.server.common.transport.queue; +package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; @@ -15,7 +16,7 @@ public interface TransportQueueProvider { TbQueueProducer> getRuleEngineMsgProducer(); - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> getTbCoreMsgProducer(); TbQueueConsumer> getTransportNotificationsConsumer(); diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/queue/src/main/proto/transport.proto similarity index 75% rename from common/transport/transport-api/src/main/proto/transport.proto rename to common/queue/src/main/proto/transport.proto index e8b513574a..807aa571bf 100644 --- a/common/transport/transport-api/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/transport.proto @@ -20,16 +20,18 @@ option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; /** - * Data Structures; + * Transport Service Data Structures; */ message SessionInfoProto { - string nodeId = 1; + string ServiceId = 1; int64 sessionIdMSB = 2; int64 sessionIdLSB = 3; int64 tenantIdMSB = 4; int64 tenantIdLSB = 5; int64 deviceIdMSB = 6; int64 deviceIdLSB = 7; + string deviceName = 8; + string deviceType = 9; } enum SessionEvent { @@ -81,7 +83,7 @@ message DeviceInfoProto { } /** - * Messages that use Data Structures; + * Transport Service Messages; */ message SessionEventMsg { SessionType sessionType = 1; @@ -181,7 +183,7 @@ message ClaimDeviceMsg { int64 durationMs = 4; } -//Used to report session state to tb-node and persist this state in the cache on the tb-node level. +//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; bool attributeSubscription = 2; @@ -200,45 +202,61 @@ message DeviceSessionsCacheEntry { message TransportToDeviceActorMsg { SessionInfoProto sessionInfo = 1; SessionEventMsg sessionEvent = 2; - PostTelemetryMsg postTelemetry = 3; - PostAttributeMsg postAttributes = 4; - GetAttributeRequestMsg getAttributes = 5; - SubscribeToAttributeUpdatesMsg subscribeToAttributes = 6; - SubscribeToRPCMsg subscribeToRPC = 7; - ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8; - ToServerRpcRequestMsg toServerRPCCallRequest = 9; - SubscriptionInfoProto subscriptionInfo = 10; - ClaimDeviceMsg claimDevice = 11; + GetAttributeRequestMsg getAttributes = 3; + SubscribeToAttributeUpdatesMsg subscribeToAttributes = 4; + SubscribeToRPCMsg subscribeToRPC = 5; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 6; + SubscriptionInfoProto subscriptionInfo = 7; + ClaimDeviceMsg claimDevice = 8; +} + +message TransportToRuleEngineMsg { + SessionInfoProto sessionInfo = 1; + PostTelemetryMsg postTelemetry = 2; + PostAttributeMsg postAttributes = 3; + ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; + ToServerRpcRequestMsg toServerRPCCallRequest = 5; + SubscriptionInfoProto subscriptionInfo = 6; } message DeviceActorToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; } /** * Main messages; */ -message ToRuleEngineMsg { - TransportToDeviceActorMsg toDeviceActorMsg = 1; -} - -message ToTransportMsg { - DeviceActorToTransportMsg toDeviceSessionMsg = 1; -} +/* Request from Transport Service to ThingsBoard Core Service */ message TransportApiRequestMsg { ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; } +/* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; } + +/* Messages that are handled by ThingsBoard Core Service */ +message ToCoreMsg { + TransportToDeviceActorMsg toDeviceActorMsg = 1; +} + +/* Messages that are handled by ThingsBoard RuleEngine Service */ +message ToRuleEngineMsg { + TransportToRuleEngineMsg toRuleEngineMsg = 1; +} + +/* Messages that are handled by ThingsBoard Transport Service */ +message ToTransportMsg { + DeviceActorToTransportMsg toDeviceSessionMsg = 1; +} diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 3920234f7c..d6dcbbabc6 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -99,19 +99,6 @@ org.apache.commons commons-lang3 - - com.google.protobuf - protobuf-java - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java deleted file mode 100644 index e16c7fd93c..0000000000 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/queue/KafkaTransportQueueProvider.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.thingsboard.server.common.transport.queue; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; - -@Component -@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") -@Slf4j -public class KafkaTransportQueueProvider implements TransportQueueProvider { - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { - return null; - } - - @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return null; - } - - @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return null; - } - - @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { - return null; - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 9ccb5369f8..e4dc2ba1f1 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -34,12 +34,15 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.queue.TransportQueueProvider; +import org.thingsboard.server.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToRuleEngineMsg; import org.thingsboard.server.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; @@ -81,7 +84,7 @@ public class DefaultTransportService implements TransportService { protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; - protected TbQueueProducer> tbCoreMsgProducer; + protected TbQueueProducer> tbCoreMsgProducer; protected TbQueueConsumer> transportNotificationsConsumer; protected ScheduledExecutorService schedulerExecutor; @@ -188,22 +191,16 @@ public class DefaultTransportService implements TransportService { if (log.isTraceEnabled()) { log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); } - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(msg).build(), callback); } @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSessionEvent(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSessionEvent(msg).build(), callback); } } @@ -211,11 +208,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostTelemetry(msg).build() - ).build(); - sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). + setPostTelemetry(msg).build(), callback); } } @@ -223,11 +217,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setPostAttributes(msg).build() - ).build(); - sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). + setPostAttributes(msg).build(), callback); } } @@ -235,11 +226,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setGetAttributes(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setGetAttributes(msg).build(), callback); } } @@ -248,11 +236,8 @@ public class DefaultTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToAttributes(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToAttributes(msg).build(), callback); } } @@ -261,11 +246,8 @@ public class DefaultTransportService implements TransportService { if (checkLimits(sessionInfo, msg, callback)) { SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo); sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscribeToRPC(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscribeToRPC(msg).build(), callback); } } @@ -273,11 +255,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToDeviceRPCCallResponse(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setToDeviceRPCCallResponse(msg).build(), callback); } } @@ -286,23 +265,16 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setToServerRPCCallRequest(msg).build() - ).build(); - sendToRuleEngine(sessionInfo, toRuleEngineMsg, callback); + sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). + setToServerRPCCallRequest(msg).build(), callback); } } @Override - public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, - TransportServiceCallback callback) { + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg( - TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setClaimDevice(msg).build() - ).build(); - sendToDeviceActor(sessionInfo, toRuleEngineMsg, callback); + sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setClaimDevice(msg).build(), callback); } } @@ -454,13 +426,15 @@ public class DefaultTransportService implements TransportService { .setEvent(event).build(); } - protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { + tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? new TransportTbQueueCallback(callback) : null); } - protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, ToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), toRuleEngineMsg), callback != null ? + protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { + ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? new TransportTbQueueCallback(callback) : null); } @@ -473,16 +447,12 @@ public class DefaultTransportService implements TransportService { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onSuccess(null); - }); + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onSuccess(null)); } @Override public void onFailure(Throwable t) { - DefaultTransportService.this.transportCallbackExecutor.submit(() -> { - callback.onError(t); - }); + DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); } } } From fa9194c1c13f3382cd7404143db2fdbb9d28aae8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 12 Mar 2020 11:11:56 +0200 Subject: [PATCH 008/196] Core consumer service implementation --- .../queue/DefaultTbCoreConsumerService.java | 126 ++++++++++++++++++ .../server/service/queue/MsgPackCallback.java | 37 +++++ .../service/queue/TbCoreConsumerService.java | 9 ++ .../server/service/queue/TbMsgCallback.java | 9 ++ .../DefaultTbCoreToTransportService.java | 72 ++++++++++ .../transport/RemoteTransportApiService.java | 3 +- .../transport/RuleEngineTransportService.java | 31 ----- .../transport/TbCoreToTransportService.java | 13 ++ .../msg/TransportToDeviceActorMsgWrapper.java | 5 +- .../src/main/resources/thingsboard.yml | 10 +- .../provider/KafkaTbCoreQueueProvider.java | 2 +- .../provider/KafkaTransportQueueProvider.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 3 + 13 files changed, 283 insertions(+), 39 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java new file mode 100644 index 0000000000..d919857318 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -0,0 +1,126 @@ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.transport.RuleEngineStats; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@Slf4j +public class DefaultTbCoreConsumerService implements TbCoreConsumerService { + + @Value("${queue.core.poll_interval}") + private long pollDuration; + @Value("${queue.core.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.core.stats.enabled:false}") + private boolean statsEnabled; + + private final ActorSystemContext actorContext; + private final TbQueueConsumer> consumer; + private final RuleEngineStats stats = new RuleEngineStats(); + private volatile ExecutorService mainConsumerExecutor; + private volatile boolean stopped = false; + + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + this.actorContext = actorContext; + } + + @PostConstruct + public void init() { + this.consumer.subscribe(); + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + ackMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); + try { + ToCoreMsg toRuleEngineMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg(), callback); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + } + consumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + @PreDestroy + public void destroy() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java new file mode 100644 index 0000000000..c700c18dc0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class MsgPackCallback implements TbMsgCallback { + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap> ackMap; + private final UUID id; + + public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, ConcurrentMap> ackMap) { + this.id = id; + this.processingTimeoutLatch = processingTimeoutLatch; + this.ackMap = ackMap; + } + + @Override + public void onSuccess() { + if (ackMap.remove(id) != null && ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } + + @Override + public void onFailure(Throwable t) { + TbProtoQueueMsg message = ackMap.remove(id); + log.warn("Failed to process message: {}", message.getValue(), t); + if (ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java new file mode 100644 index 0000000000..0541de7c63 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -0,0 +1,9 @@ +package org.thingsboard.server.service.queue; + +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.function.Consumer; + +public interface TbCoreConsumerService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java new file mode 100644 index 0000000000..44dd058e1e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -0,0 +1,9 @@ +package org.thingsboard.server.service.queue; + +public interface TbMsgCallback { + + void onSuccess(); + + void onFailure(Throwable t); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java new file mode 100644 index 0000000000..c1b0608d44 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -0,0 +1,72 @@ +package org.thingsboard.server.service.transport; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.TbQueueCallback; +import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.provider.TbCoreQueueProvider; + +import java.util.UUID; +import java.util.function.Consumer; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +public class DefaultTbCoreToTransportService implements TbCoreToTransportService { + + private final TbCoreQueueProvider tbCoreQueueProvider; + private final TbQueueProducer> tbTransportProducer; + + @Value("${queue.notifications.topic}") + private String notificationsTopic; + + public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { + this.tbCoreQueueProvider = tbCoreQueueProvider; + this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg) { + process(nodeId, msg, null, null); + } + + @Override + public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + String topic = notificationsTopic + "." + nodeId; + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); + log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); + tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + } + + private static class QueueCallbackAdaptor implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 644d4191b1..354252642f 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -22,6 +22,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; @@ -42,7 +43,7 @@ import java.util.concurrent.*; * Created by ashvayka on 05.10.18. */ @Slf4j -@Component +@Service @ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java deleted file mode 100644 index 0d401e4d9c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineTransportService.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; - -import java.util.function.Consumer; - -/** - * Created by ashvayka on 05.10.18. - */ -public interface RuleEngineTransportService { - - void process(String nodeId, DeviceActorToTransportMsg msg); - - void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java new file mode 100644 index 0000000000..3324f02440 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -0,0 +1,13 @@ +package org.thingsboard.server.service.transport; + +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.function.Consumer; + +public interface TbCoreToTransportService { + + void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg); + + void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 34abac72bb..880d7507ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.service.queue.TbMsgCallback; import java.io.Serializable; import java.util.UUID; @@ -36,9 +37,11 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; + private final TbMsgCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbMsgCallback callback) { this.msg = msg; + this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); this.deviceId = new DeviceId(new UUID(msg.getSessionInfo().getDeviceIdMSB(), msg.getSessionInfo().getDeviceIdLSB())); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index dc479191e7..2ded608203 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -466,7 +466,6 @@ js: print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" transport: - type: "${TRANSPORT_TYPE:local}" # local or remote sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -533,7 +532,6 @@ swagger: url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" version: "${SWAGGER_VERSION:2.0}" - queue: type: "${TB_QUEUE_TYPE:kafka}" kafka: @@ -555,6 +553,10 @@ queue: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" @@ -565,5 +567,5 @@ queue: notifications: topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" -rule_engine: - type: "${RULE_ENGINE_TYPE:local}" # local or remote \ No newline at end of file +service: + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport \ No newline at end of file diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index eac81b80c3..2f9a9ad8f8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -8,7 +8,7 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; @Component -@ConditionalOnExpression("'${transport.type:null}'=='local' || '${transport.type:null}'=='remote'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && ${queue.type:null}'=='kafka'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider{ @Override public TbQueueProducer> getTransportMsgProducer() { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 6686b3704f..945888a45a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -14,7 +14,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; @Component -@ConditionalOnExpression("'${transport.type:null}'=='null' || '${transport.type}'=='local'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && ${queue.type:null}'=='kafka'") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @Override diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 741adc6795..721db2605d 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -62,6 +62,9 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" +service: + type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport + queue: type: "${TB_QUEUE_TYPE:kafka}" kafka: From c4c53bfbd8ed09c9e533318f73fb55654ff5381a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 12 Mar 2020 14:37:38 +0200 Subject: [PATCH 009/196] Transport Implementation --- .../server/actors/ActorSystemContext.java | 9 +- .../device/DeviceActorMessageProcessor.java | 35 ++-- .../actors/service/DefaultActorService.java | 3 - .../cluster/rpc/ClusterGrpcService.java | 161 ------------------ .../cluster/rpc/ClusterRpcService.java | 42 ----- .../service/cluster/rpc/GrpcSession.java | 125 -------------- .../cluster/rpc/GrpcSessionListener.java | 32 ---- .../service/cluster/rpc/RpcMsgListener.java | 32 ---- .../queue/DefaultTbCoreConsumerService.java | 11 +- .../DefaultTbRuleEngineConsumerService.java | 124 ++++++++++++++ .../service/queue/TbCoreConsumerStats.java | 70 ++++++++ .../queue/TbRuleEngineConsumerService.java | 5 + .../TbRuleEngineConsumerStats.java} | 4 +- .../RemoteRuleEngineTransportService.java | 3 +- common/queue/src/main/proto/transport.proto | 2 +- 15 files changed, 229 insertions(+), 429 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java rename application/src/main/java/org/thingsboard/server/service/{transport/RuleEngineStats.java => queue/TbRuleEngineConsumerStats.java} (97%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index bf06a3a8ac..d10436075b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -67,7 +67,6 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.cluster.discovery.DiscoveryService; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -81,7 +80,7 @@ import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.service.transport.RuleEngineTransportService; +import org.thingsboard.server.service.transport.TbCoreToTransportService; import javax.annotation.Nullable; import java.io.IOException; @@ -122,10 +121,6 @@ public class ActorSystemContext { @Getter private ClusterRoutingService routingService; - @Autowired - @Getter - private ClusterRpcService rpcService; - @Autowired @Getter private DataDecodingEncodingService encodingService; @@ -241,7 +236,7 @@ public class ActorSystemContext { @Lazy @Autowired @Getter - private RuleEngineTransportService ruleEngineTransportService; + private TbCoreToTransportService tbCoreToTransportService; @Lazy @Autowired diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 2752b53090..6c0dde4044 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -234,24 +234,24 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } - if (msg.hasPostAttributes()) { - handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); - reportDeviceActivity = true; - } - if (msg.hasPostTelemetry()) { - handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); - reportDeviceActivity = true; - } +// if (msg.hasPostAttributes()) { +// handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); +// reportDeviceActivity = true; +// } +// if (msg.hasPostTelemetry()) { +// handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); +// reportDeviceActivity = true; +// } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } - if (msg.hasToServerRPCCallRequest()) { - handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); - reportDeviceActivity = true; - } +// if (msg.hasToServerRPCCallRequest()) { +// handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); +// reportDeviceActivity = true; +// } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } @@ -260,6 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } + //TODO: 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } @@ -540,7 +541,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); - systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg); } void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) { @@ -556,7 +557,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); - systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg); + systemContext.getTbCoreToTransportService().process(sessionInfo.getNodeId(), msg); } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { @@ -564,7 +565,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { @@ -572,7 +573,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { @@ -580,7 +581,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); - systemContext.getRuleEngineTransportService().process(nodeId, msg); + systemContext.getTbCoreToTransportService().process(nodeId, msg); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 43344ef374..e5da93b9e2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -52,15 +52,12 @@ import org.thingsboard.server.service.cluster.discovery.DiscoveryService; import org.thingsboard.server.service.cluster.discovery.ServerInstance; import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.transport.RuleEngineStats; import scala.concurrent.Await; import scala.concurrent.Future; import scala.concurrent.duration.Duration; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java deleted file mode 100644 index 23b050634c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterGrpcService.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import com.google.protobuf.ByteString; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.discovery.ServerInstanceService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; - -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class ClusterGrpcService extends ClusterRpcServiceGrpc.ClusterRpcServiceImplBase implements ClusterRpcService { - - @Autowired - private ServerInstanceService instanceService; - - @Autowired - private DataDecodingEncodingService encodingService; - - private RpcMsgListener listener; - - private Server server; - - private ServerInstance instance; - - private ConcurrentMap>> pendingSessionMap = - new ConcurrentHashMap<>(); - - public void init(RpcMsgListener listener) { - this.listener = listener; - log.info("Initializing RPC service!"); - instance = instanceService.getSelf(); - server = ServerBuilder.forPort(instance.getPort()).addService(this).build(); - log.info("Going to start RPC server using port: {}", instance.getPort()); - try { - server.start(); - } catch (IOException e) { - log.error("Failed to start RPC server!", e); - throw new RuntimeException("Failed to start RPC server!"); - } - log.info("RPC service initialized!"); - } - - @Override - public void onSessionCreated(UUID msgUid, StreamObserver inputStream) { - BlockingQueue> queue = pendingSessionMap.remove(msgUid); - if (queue != null) { - try { - queue.put(inputStream); - } catch (InterruptedException e) { - log.warn("Failed to report created session!"); - Thread.currentThread().interrupt(); - } - } else { - log.warn("Failed to lookup pending session!"); - } - } - - @Override - public StreamObserver handleMsgs( - StreamObserver responseObserver) { - log.info("Processing new session."); - return createSession(new RpcSessionCreateRequestMsg(UUID.randomUUID(), null, responseObserver)); - } - - - @PreDestroy - public void stop() { - if (server != null) { - log.info("Going to onStop RPC server"); - server.shutdownNow(); - try { - server.awaitTermination(); - log.info("RPC server stopped!"); - } catch (InterruptedException e) { - log.warn("Failed to onStop RPC server!"); - Thread.currentThread().interrupt(); - } - } - } - - - @Override - public void broadcast(RpcBroadcastMsg msg) { - listener.onBroadcastMsg(msg); - } - - private StreamObserver createSession(RpcSessionCreateRequestMsg msg) { - BlockingQueue> queue = new ArrayBlockingQueue<>(1); - pendingSessionMap.put(msg.getMsgUid(), queue); - listener.onRpcSessionCreateRequestMsg(msg); - try { - StreamObserver observer = queue.take(); - log.info("Processed new session."); - return observer; - } catch (Exception e) { - log.info("Failed to process session.", e); - throw new RuntimeException(e); - } - } - - @Override - public void tell(ClusterAPIProtos.ClusterMessage message) { - listener.onSendMsg(message); - } - - @Override - public void tell(ServerAddress serverAddress, TbActorMsg actorMsg) { - listener.onSendMsg(encodingService.convertToProtoDataMessage(serverAddress, actorMsg)); - } - - @Override - public void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data) { - ClusterAPIProtos.ClusterMessage msg = ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(msgType) - .setPayload(ByteString.copyFrom(data)).build(); - listener.onSendMsg(msg); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java deleted file mode 100644 index 637924fca9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/ClusterRpcService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.stub.StreamObserver; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRpcService { - - void init(RpcMsgListener listener); - - void broadcast(RpcBroadcastMsg msg); - - void onSessionCreated(UUID msgUid, StreamObserver inputStream); - - void tell(ClusterAPIProtos.ClusterMessage message); - - void tell(ServerAddress serverAddress, TbActorMsg actorMsg); - - void tell(ServerAddress serverAddress, ClusterAPIProtos.MessageType msgType, byte[] data); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java deleted file mode 100644 index aff0bc4a41..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSession.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.stub.StreamObserver; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.io.Closeable; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -@Slf4j -public final class GrpcSession implements Closeable { - private final UUID sessionId; - private final boolean client; - private final GrpcSessionListener listener; - private final ManagedChannel channel; - private StreamObserver inputStream; - private StreamObserver outputStream; - - private boolean connected; - private ServerAddress remoteServer; - - public GrpcSession(GrpcSessionListener listener) { - this(null, listener, null); - } - - public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) { - this.sessionId = UUID.randomUUID(); - this.listener = listener; - if (remoteServer != null) { - this.client = true; - this.connected = true; - this.remoteServer = remoteServer; - } else { - this.client = false; - } - this.channel = channel; - } - - public void initInputStream() { - this.inputStream = new StreamObserver() { - @Override - public void onNext(ClusterAPIProtos.ClusterMessage clusterMessage) { - if (!connected && clusterMessage.getMessageType() == ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE) { - connected = true; - ServerAddress rpcAddress = new ServerAddress(clusterMessage.getServerAddress().getHost(), clusterMessage.getServerAddress().getPort(), ServerType.CORE); - remoteServer = new ServerAddress(rpcAddress.getHost(), rpcAddress.getPort(), ServerType.CORE); - listener.onConnected(GrpcSession.this); - } - if (connected) { - listener.onReceiveClusterGrpcMsg(GrpcSession.this, clusterMessage); - } - } - - @Override - public void onError(Throwable t) { - listener.onError(GrpcSession.this, t); - } - - @Override - public void onCompleted() { - outputStream.onCompleted(); - listener.onDisconnected(GrpcSession.this); - } - }; - } - - public void initOutputStream() { - if (client) { - listener.onConnected(GrpcSession.this); - } - } - - public void sendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (connected) { - try { - outputStream.onNext(msg); - } catch (Throwable t) { - try { - outputStream.onError(t); - } catch (Throwable t2) { - } - listener.onError(GrpcSession.this, t); - } - } else { - log.warn("[{}] Failed to send message due to closed session!", sessionId); - } - } - - @Override - public void close() { - connected = false; - try { - outputStream.onCompleted(); - } catch (IllegalStateException e) { - log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); - } - if (channel != null) { - channel.shutdownNow(); - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java deleted file mode 100644 index a6ecf967d6..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/GrpcSessionListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -public interface GrpcSessionListener { - - void onConnected(GrpcSession session); - - void onDisconnected(GrpcSession session); - - void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage); - - void onError(GrpcSession session, Throwable t); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java deleted file mode 100644 index 2ff37b39e0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/rpc/RpcMsgListener.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.rpc; - -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ - -public interface RpcMsgListener { - void onReceivedMsg(ServerAddress remoteServer, ClusterAPIProtos.ClusterMessage msg); - void onSendMsg(ClusterAPIProtos.ClusterMessage msg); - void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg); - void onBroadcastMsg(RpcBroadcastMsg msg); -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d919857318..4cd5c664fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -15,7 +15,6 @@ import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.transport.RuleEngineStats; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -44,7 +43,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private final ActorSystemContext actorContext; private final TbQueueConsumer> consumer; - private final RuleEngineStats stats = new RuleEngineStats(); + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; @@ -71,10 +70,10 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { ackMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { - ToCoreMsg toRuleEngineMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg(), callback); + ToCoreMsg toCoreMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toCoreMsg); + if (toCoreMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); } else { callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java new file mode 100644 index 0000000000..96b72843ed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -0,0 +1,124 @@ +package org.thingsboard.server.service.queue; + +import akka.actor.ActorRef; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +@Slf4j +public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { + + + @Value("${queue.rule-engine.poll_interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:false}") + private boolean statsEnabled; + + private final ActorSystemContext actorContext; + private final TbQueueConsumer> consumer; + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + private volatile ExecutorService mainConsumerExecutor; + private volatile boolean stopped = false; + + public DefaultTbRuleEngineConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + this.actorContext = actorContext; + } + + @PostConstruct + public void init() { + this.consumer.subscribe(); + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + mainConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + ackMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); + try { + TransportProtos.ToCoreMsg toCoreMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toCoreMsg); + if (toCoreMsg.hasToDeviceActorMsg()) { + forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + } + consumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + private void forwardToDeviceActor(TransportProtos.TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + stats.printStats(); + } + } + + @PreDestroy + public void destroy() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java new file mode 100644 index 0000000000..e3bb3a3c4b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbCoreConsumerStats { + + private final AtomicInteger totalCounter = new AtomicInteger(0); + private final AtomicInteger sessionEventCounter = new AtomicInteger(0); + private final AtomicInteger getAttributesCounter = new AtomicInteger(0); + private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); + private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); + private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); + private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); + private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + + public void log(TransportProtos.TransportToDeviceActorMsg msg) { + totalCounter.incrementAndGet(); + if (msg.hasSessionEvent()) { + sessionEventCounter.incrementAndGet(); + } + if (msg.hasGetAttributes()) { + getAttributesCounter.incrementAndGet(); + } + if (msg.hasSubscribeToAttributes()) { + subscribeToAttributesCounter.incrementAndGet(); + } + if (msg.hasSubscribeToRPC()) { + subscribeToRPCCounter.incrementAndGet(); + } + if (msg.hasToDeviceRPCCallResponse()) { + toDeviceRPCCallResponseCounter.incrementAndGet(); + } + if (msg.hasSubscriptionInfo()) { + subscriptionInfoCounter.incrementAndGet(); + } + if (msg.hasClaimDevice()) { + claimDeviceCounter.incrementAndGet(); + } + } + + public void printStats() { + int total = totalCounter.getAndSet(0); + if (total > 0) { + log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]", + total, sessionEventCounter.getAndSet(0), + getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), + subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java new file mode 100644 index 0000000000..691a54aaab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -0,0 +1,5 @@ +package org.thingsboard.server.service.queue; + +public interface TbRuleEngineConsumerService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index dea6e6d33c..80d05628f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RuleEngineStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; @@ -21,7 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -public class RuleEngineStats { +public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java index 83353cea4b..980f5e6964 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.TbCoreConsumerStats; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -109,7 +110,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ private volatile boolean stopped = false; - private final RuleEngineStats stats = new RuleEngineStats(); + private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); @PostConstruct public void init() { diff --git a/common/queue/src/main/proto/transport.proto b/common/queue/src/main/proto/transport.proto index 807aa571bf..892139a84a 100644 --- a/common/queue/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/transport.proto @@ -23,7 +23,7 @@ option java_outer_classname = "TransportProtos"; * Transport Service Data Structures; */ message SessionInfoProto { - string ServiceId = 1; + string nodeId = 1; int64 sessionIdMSB = 2; int64 sessionIdLSB = 3; int64 tenantIdMSB = 4; From dc6081d820b236a2ad963b8a7c2bcd1c54bfd89a Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Fri, 13 Mar 2020 17:31:58 +0200 Subject: [PATCH 010/196] Created kafka and in-memory queue providers * Created kafka and in-memory queue providers * Added header requestTime * refactored --- .../server/actors/service/ActorService.java | 1 - .../queue/DefaultTbCoreConsumerService.java | 15 ++ .../DefaultTbRuleEngineConsumerService.java | 15 ++ .../server/service/queue/MsgPackCallback.java | 15 ++ .../service/queue/TbCoreConsumerService.java | 15 ++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../server/service/queue/TbMsgCallback.java | 15 ++ .../queue/TbRuleEngineConsumerService.java | 15 ++ .../queue/TbRuleEngineConsumerStats.java | 24 +- .../service/script/RemoteJsInvokeService.java | 8 - .../DefaultTbCoreToTransportService.java | 19 +- .../RemoteRuleEngineTransportService.java | 37 +-- .../transport/RemoteTransportApiService.java | 2 +- .../transport/TbCoreToTransportService.java | 15 ++ .../org/thingsboard/server/TbQueueAdmin.java | 15 ++ .../thingsboard/server/TbQueueCallback.java | 15 ++ .../thingsboard/server/TbQueueConsumer.java | 15 ++ ...xtractor.java => TbQueueCoreSettings.java} | 15 +- .../org/thingsboard/server/TbQueueMsg.java | 15 ++ .../thingsboard/server/TbQueueMsgHeaders.java | 15 ++ .../server/TbQueueMsgMetadata.java | 15 ++ .../thingsboard/server/TbQueueProducer.java | 21 +- .../server/TbQueueRequestTemplate.java | 15 ++ .../server/TbQueueResponseTemplate.java | 15 ++ .../server/TbQueueTransportApiSettings.java | 40 ++++ .../common/AbstractTbQueueTemplate.java | 29 +++ .../common/DefaultTbQueueMsgHeaders.java | 15 ++ .../common/DefaultTbQueueRequestTemplate.java | 18 +- .../DefaultTbQueueResponseTemplate.java | 88 +++++--- .../server/common/TbProtoQueueMsg.java | 15 ++ .../server/kafka/AbstractTbKafkaTemplate.java | 50 ---- .../server/kafka/AsyncCallbackTemplate.java | 66 ------ .../server/kafka/KafkaTbQueueMsg.java | 15 ++ .../server/kafka/KafkaTbQueueMsgMetadata.java | 15 ++ .../server/kafka/TBKafkaAdmin.java | 32 ++- .../server/kafka/TBKafkaConsumerTemplate.java | 20 -- .../server/kafka/TBKafkaProducerTemplate.java | 44 +--- .../server/kafka/TbKafkaRequestTemplate.java | 213 ------------------ .../server/kafka/TbKafkaResponseTemplate.java | 161 ------------- .../server/kafka/TbKafkaSettings.java | 12 +- .../server/memory/InMemoryStorage.java | 15 ++ .../memory/InMemoryTbQueueConsumer.java | 15 ++ .../memory/InMemoryTbQueueProducer.java | 30 ++- .../provider/InMemoryTbCoreQueueProvider.java | 79 +++++++ .../InMemoryTransportQueueProvider.java | 79 +++++++ .../provider/KafkaTbCoreQueueProvider.java | 88 ++++++-- .../provider/KafkaTransportQueueProvider.java | 70 +++++- .../server/provider/TbCoreQueueProvider.java | 17 +- .../provider/TransportQueueProvider.java | 15 ++ .../service/DefaultTransportService.java | 2 +- 50 files changed, 905 insertions(+), 692 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/{kafka/TbKafkaRequestIdExtractor.java => TbQueueCoreSettings.java} (64%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 3032b1dfe0..48a31127f4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 4cd5c664fd..0d0e38a1b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 96b72843ed..413d5ee9dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index c700c18dc0..5b94db28b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 0541de7c63..1e1b352645 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index e3bb3a3c4b..f7912836c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java index 44dd058e1e..774377407a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbMsgCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 691a54aaab..671cd72262 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbRuleEngineConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 80d05628f2..75711d20b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -25,13 +25,13 @@ public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); private final AtomicInteger sessionEventCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); +// private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); +// private final AtomicInteger postAttributesCounter = new AtomicInteger(0); private final AtomicInteger getAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); +// private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); @@ -40,12 +40,12 @@ public class TbRuleEngineConsumerStats { if (msg.hasSessionEvent()) { sessionEventCounter.incrementAndGet(); } - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } +// if (msg.hasPostTelemetry()) { +// postTelemetryCounter.incrementAndGet(); +// } +// if (msg.hasPostAttributes()) { +// postAttributesCounter.incrementAndGet(); +// } if (msg.hasGetAttributes()) { getAttributesCounter.incrementAndGet(); } @@ -58,9 +58,9 @@ public class TbRuleEngineConsumerStats { if (msg.hasToDeviceRPCCallResponse()) { toDeviceRPCCallResponseCounter.incrementAndGet(); } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); - } +// if (msg.hasToServerRPCCallRequest()) { +// toServerRPCCallRequestCounter.incrementAndGet(); +// } if (msg.hasSubscriptionInfo()) { subscriptionInfoCounter.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index d39465ff12..08578c3890 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; @@ -29,7 +28,6 @@ import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbNodeIdProvider; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -45,12 +43,6 @@ import java.util.concurrent.atomic.AtomicInteger; @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Autowired - private TbNodeIdProvider nodeIdProvider; - - @Autowired - private TbKafkaSettings kafkaSettings; - @Value("${js.remote.request_topic}") private String requestTopic; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index c1b0608d44..fd8ee406c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; @@ -15,6 +30,8 @@ import org.thingsboard.server.provider.TbCoreQueueProvider; import java.util.UUID; import java.util.function.Consumer; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; + @Slf4j @Service @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") @@ -42,7 +59,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java index 980f5e6964..eed2b8fa8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java @@ -41,9 +41,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTranspo import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.provider.TbCoreQueueProvider; import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.TbCoreConsumerStats; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; @@ -65,7 +64,7 @@ import java.util.function.Consumer; @Slf4j @Service @ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService implements RuleEngineTransportService { +public class RemoteRuleEngineTransportService { @Value("${transport.remote.rule_engine.topic}") private String ruleEngineTopic; @@ -85,12 +84,6 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Value("${transport.remote.rule_engine.stats.enabled:false}") private boolean statsEnabled; -// @Autowired -// private TbKafkaSettings kafkaSettings; -// - @Autowired - private TbNodeIdProvider nodeIdProvider; - @Autowired private ActorSystemContext actorContext; @@ -102,6 +95,9 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @Autowired private DataDecodingEncodingService encodingService; + @Autowired + private TbCoreQueueProvider coreQueueProvider; + private TbQueueConsumer> ruleEngineConsumer; private TbQueueProducer> notificationsProducer; @@ -114,26 +110,12 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ @PostConstruct public void init() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder notificationsProducerBuilder = TBKafkaProducerTemplate.builder(); - notificationsProducerBuilder.settings(kafkaSettings); - notificationsProducerBuilder.clientId("producer-transport-notification-" + nodeIdProvider.getNodeId()); - notificationsProducerBuilder.encoder(new ToTransportMsgEncoder()); - - notificationsProducer = notificationsProducerBuilder.build(); + notificationsProducer = coreQueueProvider.getTransportMsgProducer(); notificationsProducer.init(); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder ruleEngineConsumerBuilder = TBKafkaConsumerTemplate.builder(); - ruleEngineConsumerBuilder.settings(kafkaSettings); - ruleEngineConsumerBuilder.topic(ruleEngineTopic); - ruleEngineConsumerBuilder.clientId("transport-" + nodeIdProvider.getNodeId()); - ruleEngineConsumerBuilder.groupId("tb-node"); - ruleEngineConsumerBuilder.autoCommit(true); - ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval); - ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize); - ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder()); - - ruleEngineConsumer = ruleEngineConsumerBuilder.build(); - ruleEngineConsumer.subscribe(); + //TODO: 2.5 +// ruleEngineConsumer = +// ruleEngineConsumer.subscribe(); } @EventListener(ApplicationReadyEvent.class) @@ -198,6 +180,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); + //TODO: 2.5 id TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 354252642f..4765737430 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 3324f02440..5b11edb5f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java index 7f8cff5f22..6917c23719 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java index 3d7d791ae4..e823d2a5fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueCallback { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index ca51eb4ddb..ddf9d7d9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java similarity index 64% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java rename to common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java index 6e433e71a9..dc84625634 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestIdExtractor.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java @@ -13,12 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server; -import java.util.UUID; +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; -public interface TbKafkaRequestIdExtractor { +@Data +@Component +public class TbQueueCoreSettings { - UUID extractRequestId(T value); + @Value("${queue.core.topic}") + private String topic; + @Value("${queue.core.partitions}") + private int partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java index 22714af77e..ca8f3a61da 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java index 9dad87588d..f95454c976 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java index 8eecae51fb..a3331999a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueMsgMetadata { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 41d165d743..206f753168 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -1,15 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; - public interface TbQueueProducer { void init(); String getDefaultTopic(); - ListenableFuture send(T msg, TbQueueCallback callback); + void send(T msg, TbQueueCallback callback); - ListenableFuture send(String topic, T msg, TbQueueCallback callback); + void send(String topic, T msg, TbQueueCallback callback); } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 5ba28f22c5..5182bb7260 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java index f3b4e4ad5d..686570ca32 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueResponseTemplate { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java new file mode 100644 index 0000000000..603d787fa8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueTransportApiSettings { + @Value("${queue.transport_api.requests_topic}") + private String requestsTopic; + + @Value("${queue.transport_api.responses_topic}") + private String responsesTopic; + + @Value("${queue.transport_api.max_pending_requests}") + private int maxPendingRequests; + + @Value("${queue.transport_api.max_requests_timeout}") + private int maxRequestsTimeout; + + @Value("${queue.transport_api.response_poll_interval}") + private long responsePollInterval; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java index 2f406a9631..b70fc417c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import java.nio.ByteBuffer; @@ -7,6 +22,7 @@ import java.util.UUID; public class AbstractTbQueueTemplate { protected static final String REQUEST_ID_HEADER = "requestId"; protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; + protected static final String REQUEST_TIME = "requestTime"; protected byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); @@ -29,4 +45,17 @@ public class AbstractTbQueueTemplate { protected String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } + + private static ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); + + protected static byte[] longToBytes(long x) { + longBuffer.putLong(0, x); + return longBuffer.array(); + } + + protected static long bytesToLong(byte[] bytes) { + longBuffer.put(bytes, 0, bytes.length); + longBuffer.flip();//need flip + return longBuffer.getLong(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java index e547a6c070..de0c368a65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import org.thingsboard.server.TbQueueMsgHeaders; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 49022ad2e3..d022dafadc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import com.google.common.util.concurrent.Futures; @@ -79,7 +94,7 @@ public class DefaultTbQueueRequestTemplate { log.trace("Received response to Kafka Template request: {}", response); byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); - UUID requestId = null; + UUID requestId; if (requestIdHeader == null) { log.error("[{}] Missing requestId in header and body", response); } else { @@ -140,6 +155,7 @@ public class DefaultTbQueueRequestTemplate future = SettableFuture.create(); ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 92dac0551b..3804fb5160 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Builder; @@ -70,41 +85,46 @@ public class DefaultTbQueueResponseTemplate requests = requestTemplate.poll(pollInterval); + requests.forEach(request -> { - byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader); - String responseTopic = bytesToString(responseTopicHeader); - try { - pendingRequestCount.getAndIncrement(); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), - response -> { - pendingRequestCount.decrementAndGet(); - response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(responseTopic, response, null); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timeout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); + long currentTime = System.currentTimeMillis(); + long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); + if (requestTime + requestTimeout >= currentTime) { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseTemplate.send(responseTopic, response, null); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + }, + requestTimeout, + timeoutExecutor, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + } } }); requestTemplate.commit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java index c072dc4d67..8dd492504c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Data; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java deleted file mode 100644 index 0c68c8dd5d..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AbstractTbKafkaTemplate.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.extern.slf4j.Slf4j; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public abstract class AbstractTbKafkaTemplate { - protected byte[] uuidToBytes(UUID uuid) { - ByteBuffer buf = ByteBuffer.allocate(16); - buf.putLong(uuid.getMostSignificantBits()); - buf.putLong(uuid.getLeastSignificantBits()); - return buf.array(); - } - - protected static UUID bytesToUuid(byte[] bytes) { - ByteBuffer bb = ByteBuffer.wrap(bytes); - long firstLong = bb.getLong(); - long secondLong = bb.getLong(); - return new UUID(firstLong, secondLong); - } - - protected byte[] stringToBytes(String string) { - return string.getBytes(StandardCharsets.UTF_8); - } - - protected String bytesToString(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java deleted file mode 100644 index 17599bfccb..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/AsyncCallbackTemplate.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -import java.util.concurrent.Executor; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 05.10.18. - */ -public class AsyncCallbackTemplate { - - public static void withCallbackAndTimeout(ListenableFuture future, - Consumer onSuccess, - Consumer onFailure, - long timeoutInMs, - ScheduledExecutorService timeoutExecutor, - Executor callbackExecutor) { - future = Futures.withTimeout(future, timeoutInMs, TimeUnit.MILLISECONDS, timeoutExecutor); - withCallback(future, onSuccess, onFailure, callbackExecutor); - } - - public static void withCallback(ListenableFuture future, Consumer onSuccess, - Consumer onFailure, Executor executor) { - FutureCallback callback = new FutureCallback() { - @Override - public void onSuccess(T result) { - try { - onSuccess.accept(result); - } catch (Throwable th) { - onFailure(th); - } - } - - @Override - public void onFailure(Throwable t) { - onFailure.accept(t); - } - }; - if (executor != null) { - Futures.addCallback(future, callback, executor); - } else { - Futures.addCallback(future, callback); - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java index e44a93a70d..9f8e73bdfc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import org.apache.kafka.clients.consumer.ConsumerRecord; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java index 4698c226d6..09cc292deb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import lombok.AllArgsConstructor; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java index 9c6d911d33..0bb3a75db6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -15,16 +15,17 @@ */ package org.thingsboard.server.kafka; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.JdkFutureAdapters; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.kafka.clients.admin.*; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; import org.thingsboard.server.TbQueueAdmin; -import java.time.Duration; import java.util.Collections; -import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -40,6 +41,19 @@ public class TBKafkaAdmin implements TbQueueAdmin { client = AdminClient.create(settings.toProps()); } + @Override + public ListenableFuture createTopicIfNotExists(String topic) { + + KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); + + ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); + + return Futures.transformAsync(topicFuture, topicDescription -> { + KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); + return JdkFutureAdapters.listenInPoolThread(resultFuture); + }); + } + public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { synchronized (this) { long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); @@ -54,7 +68,7 @@ public class TBKafkaAdmin implements TbQueueAdmin { } } - public CreateTopicsResult createTopic(NewTopic topic){ + public CreateTopicsResult createTopic(NewTopic topic) { return client.createTopics(Collections.singletonList(topic)); } @@ -67,10 +81,4 @@ public class TBKafkaAdmin implements TbQueueAdmin { return false; } } - - @Override - public ListenableFuture createTopicIfNotExists(String topic) { - - return null; - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 6f0ce7b55c..558d556d09 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.UUID; /** * Created by ashvayka on 24.09.18. @@ -42,15 +41,11 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; - @Builder.Default - private TbKafkaRequestIdExtractor requestIdExtractor = ((response) -> null); - @Getter private final String topic; @Builder private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, - TbKafkaRequestIdExtractor requestIdExtractor, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords) { @@ -68,7 +63,6 @@ public class TBKafkaConsumerTemplate implements TbQueueCon } this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; - this.requestIdExtractor = requestIdExtractor; this.topic = topic; } @@ -105,22 +99,8 @@ public class TBKafkaConsumerTemplate implements TbQueueCon consumer.unsubscribe(); } -// public void subscribe() { -// consumer.subscribe(Collections.singletonList(topic)); -// } -// - -// -// public ConsumerRecords poll(Duration duration) { -// return consumer.poll(duration); -// } - public T decode(ConsumerRecord record) throws IOException { return decoder.decode(new KafkaTbQueueMsg(record)); } - public UUID extractRequestId(T value) { - return requestIdExtractor.extractRequestId(value); - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 938b278207..d76a2dc625 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -15,9 +15,6 @@ */ package org.thingsboard.server.kafka; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.JdkFutureAdapters; -import com.google.common.util.concurrent.ListenableFuture; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -31,7 +28,6 @@ import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import java.util.List; @@ -91,12 +87,12 @@ public class TBKafkaProducerTemplate implements TbQueuePro } @Override - public ListenableFuture send(T msg, TbQueueCallback callback) { - return send(defaultTopic, msg, callback); + public void send(T msg, TbQueueCallback callback) { + send(defaultTopic, msg, callback); } @Override - public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + public void send(String topic, T msg, TbQueueCallback callback) { String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; @@ -111,42 +107,8 @@ public class TBKafkaProducerTemplate implements TbQueuePro callback.onFailure(exception); } }); - - return Futures.transform(JdkFutureAdapters.listenInPoolThread(result), metadata -> new KafkaTbQueueMsgMetadata(metadata)); } -// public Future send(String key, T value, Callback callback) { -// return send(key, value, null, callback); -// } -// -// public Future send(String key, T value, Iterable
headers, Callback callback) { -// return send(key, value, null, headers, callback); -// } -// -// public Future send(String key, T value, Long timestamp, Iterable
headers, Callback callback) { -// if (!StringUtils.isEmpty(this.defaultTopic)) { -// return send(this.defaultTopic, key, value, timestamp, headers, callback); -// } else { -// throw new RuntimeException("Failed to send message! Default topic is not specified!"); -// } -// } -// -// public Future send(String topic, String key, T value, Iterable
headers, Callback callback) { -// return send(topic, key, value, null, headers, callback); -// } -// -// public Future send(String topic, String key, T value, Callback callback) { -// return send(topic, key, value, null, null, callback); -// } -// -// public Future send(String topic, String key, T value, Long timestamp, Iterable
headers, Callback callback) { -// byte[] data = encoder.encode(value); -// ProducerRecord record; -// Integer partition = getPartition(topic, key, value, data); -// record = new ProducerRecord<>(topic, partition, timestamp, key, data, headers); -// return producer.send(record, callback); -// } - private Integer getPartition(String topic, T value) { if (partitioner == null) { return null; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java deleted file mode 100644 index 2a7bca119f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.CreateTopicsResult; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.TopicExistsException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeoutException; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public class TbKafkaRequestTemplate extends AbstractTbKafkaTemplate { - - private final TBKafkaProducerTemplate requestTemplate; - private final TBKafkaConsumerTemplate responseTemplate; - private final ConcurrentMap> pendingRequests; - private final boolean internalExecutor; - private final ExecutorService executor; - private final long maxRequestTimeout; - private final long maxPendingRequests; - private final long pollInterval; - private volatile long tickTs = 0L; - private volatile long tickSize = 0L; - private volatile boolean stopped = false; - - @Builder - public TbKafkaRequestTemplate(TBKafkaProducerTemplate requestTemplate, - TBKafkaConsumerTemplate responseTemplate, - long maxRequestTimeout, - long maxPendingRequests, - long pollInterval, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxRequestTimeout = maxRequestTimeout; - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - if (executor != null) { - internalExecutor = false; - this.executor = executor; - } else { - internalExecutor = true; - this.executor = Executors.newSingleThreadExecutor(); - } - } - - public void init() { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.requestTemplate.getSettings()); - CreateTopicsResult result = admin.createTopic(new NewTopic(responseTemplate.getTopic(), 1, (short) 1)); - result.all().get(); - } catch (Exception e) { - if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) { - log.trace("[{}] Topic already exists. ", responseTemplate.getTopic()); - } else { - log.info("[{}] Failed to create topic: {}", responseTemplate.getTopic(), e.getMessage(), e); - throw new RuntimeException(e); - } - - } - this.requestTemplate.init(); - tickTs = System.currentTimeMillis(); - responseTemplate.subscribe(); - executor.submit(() -> { - long nextCleanupMs = 0L; - while (!stopped) { - try { - ConsumerRecords responses = responseTemplate.poll(Duration.ofMillis(pollInterval)); - if (responses.count() > 0) { - log.trace("Polling responses completed, consumer records count [{}]", responses.count()); - } - responses.forEach(response -> { - log.trace("Received response to Kafka Template request: {}", response); - Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - Response decodedResponse = null; - UUID requestId = null; - if (requestIdHeader == null) { - try { - decodedResponse = responseTemplate.decode(response); - requestId = responseTemplate.extractRequestId(decodedResponse); - } catch (IOException e) { - log.error("Failed to decode response", e); - } - } else { - requestId = bytesToUuid(requestIdHeader.value()); - } - if (requestId == null) { - log.error("[{}] Missing requestId in header and body", response); - } else { - log.trace("[{}] Response received", requestId); - ResponseMetaData expectedResponse = pendingRequests.remove(requestId); - if (expectedResponse == null) { - log.trace("[{}] Invalid or stale request", requestId); - } else { - try { - if (decodedResponse == null) { - decodedResponse = responseTemplate.decode(response); - } - expectedResponse.future.set(decodedResponse); - } catch (IOException e) { - expectedResponse.future.setException(e); - } - } - } - }); - tickTs = System.currentTimeMillis(); - tickSize = pendingRequests.size(); - if (nextCleanupMs < tickTs) { - //cleanup; - pendingRequests.forEach((key, value) -> { - if (value.expTime < tickTs) { - ResponseMetaData staleRequest = pendingRequests.remove(key); - if (staleRequest != null) { - log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs); - staleRequest.future.setException(new TimeoutException()); - } - } - }); - nextCleanupMs = tickTs + maxRequestTimeout; - } - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } - } catch (Throwable e) { - log.warn("Failed to obtain responses from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new responses", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (internalExecutor) { - executor.shutdownNow(); - } - } - - public ListenableFuture post(String key, Request request) { - if (tickSize > maxPendingRequests) { - return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); - } - UUID requestId = UUID.randomUUID(); - List
headers = new ArrayList<>(2); - headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))); - headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic()))); - SettableFuture future = SettableFuture.create(); - ResponseMetaData responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); - pendingRequests.putIfAbsent(requestId, responseMetaData); - log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime); - requestTemplate.send(key, request, headers, (metadata, exception) -> { - if (exception != null) { - log.trace("[{}] Failed to post the request", requestId, exception); - } else { - log.trace("[{}] Posted the request: {}", requestId, metadata); - } - }); - return future; - } - - private static class ResponseMetaData { - private final long expTime; - private final SettableFuture future; - - ResponseMetaData(long ts, SettableFuture future) { - this.expTime = ts; - this.future = future; - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java deleted file mode 100644 index 864182a552..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaResponseTemplate.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Builder; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.internals.RecordHeader; - -import java.time.Duration; -import java.util.Collections; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Created by ashvayka on 25.09.18. - */ -@Slf4j -public class TbKafkaResponseTemplate extends AbstractTbKafkaTemplate { - - private final TBKafkaConsumerTemplate requestTemplate; - private final TBKafkaProducerTemplate responseTemplate; - private final TbKafkaHandler handler; - private final ConcurrentMap pendingRequests; - private final ExecutorService loopExecutor; - private final ScheduledExecutorService timeoutExecutor; - private final ExecutorService callbackExecutor; - private final int maxPendingRequests; - private final long requestTimeout; - - private final long pollInterval; - private volatile boolean stopped = false; - private final AtomicInteger pendingRequestCount = new AtomicInteger(); - - @Builder - public TbKafkaResponseTemplate(TBKafkaConsumerTemplate requestTemplate, - TBKafkaProducerTemplate responseTemplate, - TbKafkaHandler handler, - long pollInterval, - long requestTimeout, - int maxPendingRequests, - ExecutorService executor) { - this.requestTemplate = requestTemplate; - this.responseTemplate = responseTemplate; - this.handler = handler; - this.pendingRequests = new ConcurrentHashMap<>(); - this.maxPendingRequests = maxPendingRequests; - this.pollInterval = pollInterval; - this.requestTimeout = requestTimeout; - this.callbackExecutor = executor; - this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(); - this.loopExecutor = Executors.newSingleThreadExecutor(); - } - - public void init() { - this.responseTemplate.init(); - requestTemplate.subscribe(); - loopExecutor.submit(() -> { - while (!stopped) { - try { - while (pendingRequestCount.get() >= maxPendingRequests) { - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e) { - log.trace("Failed to wait until the server has capacity to handle new requests", e); - } - } - ConsumerRecords requests = requestTemplate.poll(Duration.ofMillis(pollInterval)); - requests.forEach(request -> { - Header requestIdHeader = request.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER); - if (requestIdHeader == null) { - log.error("[{}] Missing requestId in header", request); - return; - } - UUID requestId = bytesToUuid(requestIdHeader.value()); - if (requestId == null) { - log.error("[{}] Missing requestId in header and body", request); - return; - } - Header responseTopicHeader = request.headers().lastHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER); - if (responseTopicHeader == null) { - log.error("[{}] Missing response topic in header", request); - return; - } - String responseTopic = bytesToString(responseTopicHeader.value()); - try { - pendingRequestCount.getAndIncrement(); - Request decodedRequest = requestTemplate.decode(request); - AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(decodedRequest), - response -> { - pendingRequestCount.decrementAndGet(); - reply(requestId, responseTopic, response); - }, - e -> { - pendingRequestCount.decrementAndGet(); - if (e.getCause() != null && e.getCause() instanceof TimeoutException) { - log.warn("[{}] Timedout to process the request: {}", requestId, request, e); - } else { - log.trace("[{}] Failed to process the request: {}", requestId, request, e); - } - }, - requestTimeout, - timeoutExecutor, - callbackExecutor); - } catch (Throwable e) { - pendingRequestCount.decrementAndGet(); - log.warn("[{}] Failed to process the request: {}", requestId, request, e); - } - }); - } catch (InterruptException ie) { - if (!stopped) { - log.warn("Fetching data from kafka was interrupted.", ie); - } - } catch (Throwable e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - public void stop() { - stopped = true; - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - if (loopExecutor != null) { - loopExecutor.shutdownNow(); - } - } - - private void reply(UUID requestId, String topic, Response response) { - responseTemplate.send(topic, requestId.toString(), response, Collections.singletonList(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId))), null); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java index 6446643a18..566766fee6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java @@ -35,22 +35,22 @@ public class TbKafkaSettings { static final String REQUEST_ID_HEADER = "requestId"; static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - @Value("${kafka.bootstrap.servers}") + @Value("${queue.kafka.bootstrap.servers}") private String servers; - @Value("${kafka.acks}") + @Value("${queue.kafka.acks}") private String acks; - @Value("${kafka.retries}") + @Value("${queue.queue.kafka.retries}") private int retries; - @Value("${kafka.batch.size}") + @Value("${queue.kafka.batch.size}") private int batchSize; - @Value("${kafka.linger.ms}") + @Value("${queue.kafka.linger.ms}") private long lingerMs; - @Value("${kafka.buffer.memory}") + @Value("${queue.kafka.buffer.memory}") private long bufferMemory; @Value("${kafka.other:#{null}}") diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java index abf30667b5..ded4cd9810 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java index a0a35eefb8..b8ddcff89c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 32b9399164..e71d579c7f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -1,11 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; @Data @@ -15,6 +27,10 @@ public class InMemoryTbQueueProducer implements TbQueuePro private final String defaultTopic; + public InMemoryTbQueueProducer(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + @Override public void init() { @@ -26,20 +42,18 @@ public class InMemoryTbQueueProducer implements TbQueuePro } @Override - public ListenableFuture send(T msg, TbQueueCallback callback) { - return send(defaultTopic, msg, callback); + public void send(T msg, TbQueueCallback callback) { + send(defaultTopic, msg, callback); } @Override - public ListenableFuture send(String topic, T msg, TbQueueCallback callback) { + public void send(String topic, T msg, TbQueueCallback callback) { boolean result = storage.put(topic, msg); if (result) { callback.onSuccess(null); - return Futures.immediateCheckedFuture(null); } else { Exception e = new RuntimeException("Failure add msg to InMemoryQueue"); callback.onFailure(e); - return Futures.immediateFailedFuture(e); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java new file mode 100644 index 0000000000..79f5dd7da0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.memory.InMemoryTbQueueProducer; + +@Slf4j +@Component +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='in-memory'") +public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbQueueCoreSettings coreSettings; + + public InMemoryTbCoreQueueProvider(TbQueueCoreSettings coreSettings) { + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return producer; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java new file mode 100644 index 0000000000..789ea1cf5a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.memory.InMemoryTbQueueProducer; + +@Component +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='in-memory'") +@Slf4j +public class InMemoryTransportQueueProvider implements TransportQueueProvider { + + private final TbQueueTransportApiSettings transportApiSettings; + + public InMemoryTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings) { + this.transportApiSettings = transportApiSettings; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.requestTemplate(producer); + templateBuilder.responseTemplate(consumer); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return producer; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return producer; + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); + return consumer; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 2f9a9ad8f8..d830cff8fa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -1,42 +1,102 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && ${queue.type:null}'=='kafka'") -public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider{ +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='kafka'") +public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + @Override - public TbQueueProducer> getTransportMsgProducer() { - return null; + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { - return null; + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { - return null; + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { - return null; + public TbQueueConsumer> getToCoreMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO: 2.5 +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new TransportApiResponseDecoder()); + return responseBuilder.build(); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> getTransportApiRequestConsumer() { return null; } @Override - public TbQueueProducer> getTransportApiResponseProducer() { - return null; + public TbQueueProducer> getTransportApiResponseProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 945888a45a..3b671a3adb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import lombok.extern.slf4j.Slf4j; @@ -6,30 +21,77 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && ${queue.type:null}'=='kafka'") +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='kafka'") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueTransportApiSettings transportApiSettings; + + public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueTransportApiSettings transportApiSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.transportApiSettings = transportApiSettings; + } + @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { - return null; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportApiSettings.getResponsesTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO: 2.5 +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); +// responseBuilder.decoder(new TransportApiResponseDecoder()); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.requestTemplate(requestBuilder.build()); + templateBuilder.responseTemplate(responseBuilder.build()); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); } @Override public TbQueueProducer> getRuleEngineMsgProducer() { - return null; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + return requestBuilder.build(); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - return null; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + return requestBuilder.build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index b7e210f0f6..6123fc0859 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -1,9 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueResponseTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java index 14d8efd534..de69e1819a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index e4dc2ba1f1..cdf6b3f763 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From e50796d84efbe572bb7f191313791083f6d1ffd9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 13 Mar 2020 20:33:17 +0200 Subject: [PATCH 011/196] Service Discovery improvements --- .../queue/DefaultTbCoreConsumerService.java | 15 ++ .../DefaultTbRuleEngineConsumerService.java | 15 ++ .../server/service/queue/MsgPackCallback.java | 15 ++ .../service/queue/TbCoreConsumerService.java | 15 ++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../server/service/queue/TbMsgCallback.java | 15 ++ .../queue/TbRuleEngineConsumerService.java | 15 ++ .../DefaultTbCoreToTransportService.java | 15 ++ .../transport/RemoteTransportApiService.java | 2 +- .../transport/TbCoreToTransportService.java | 15 ++ .../src/main/resources/thingsboard.yml | 12 +- common/queue/pom.xml | 16 +- .../org/thingsboard/server/TbQueueAdmin.java | 15 ++ .../thingsboard/server/TbQueueCallback.java | 15 ++ .../thingsboard/server/TbQueueConsumer.java | 15 ++ .../org/thingsboard/server/TbQueueMsg.java | 15 ++ .../thingsboard/server/TbQueueMsgHeaders.java | 15 ++ .../server/TbQueueMsgMetadata.java | 15 ++ .../thingsboard/server/TbQueueProducer.java | 18 ++ .../server/TbQueueRequestTemplate.java | 15 ++ .../server/TbQueueResponseTemplate.java | 15 ++ .../common/AbstractTbQueueTemplate.java | 15 ++ .../common/DefaultTbQueueMsgHeaders.java | 15 ++ .../common/DefaultTbQueueRequestTemplate.java | 15 ++ .../DefaultTbQueueResponseTemplate.java | 15 ++ .../server/common/TbProtoQueueMsg.java | 15 ++ .../DefaultTbServiceInfoProvider.java | 90 +++++++ .../discovery/PartitionChangeEvent.java | 33 +++ .../discovery/PartitionDiscoveryService.java | 29 ++ .../server/discovery/ServiceType.java | 20 ++ .../discovery/TbServiceInfoProvider.java | 30 +++ .../server/discovery/TopicPartitionInfo.java | 26 ++ .../ZkPartitionDiscoveryService.java | 255 ++++++++++++++++++ .../server/kafka/KafkaTbQueueMsg.java | 15 ++ .../server/kafka/KafkaTbQueueMsgMetadata.java | 15 ++ .../server/memory/InMemoryStorage.java | 15 ++ .../memory/InMemoryTbQueueConsumer.java | 15 ++ .../memory/InMemoryTbQueueProducer.java | 15 ++ .../provider/KafkaTbCoreQueueProvider.java | 15 ++ .../provider/KafkaTransportQueueProvider.java | 15 ++ .../server/provider/TbCoreQueueProvider.java | 15 ++ .../provider/TransportQueueProvider.java | 15 ++ common/queue/src/main/proto/transport.proto | 10 + .../service/DefaultTransportService.java | 2 +- 44 files changed, 981 insertions(+), 14 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 4cd5c664fd..0d0e38a1b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 96b72843ed..413d5ee9dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import akka.actor.ActorRef; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index c700c18dc0..5b94db28b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 0541de7c63..1e1b352645 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index e3bb3a3c4b..f7912836c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java index 44dd058e1e..774377407a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbMsgCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 691a54aaab..671cd72262 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; public interface TbRuleEngineConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index c1b0608d44..49031f9f53 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 354252642f..4765737430 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 3324f02440..5b11edb5f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2ded608203..da2e5beb98 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -70,11 +70,6 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -# RPC connection parameters. Used only in cluster mode only. -rpc: - bind_host: "${RPC_HOST:localhost}" - bind_port: "${RPC_PORT:9001}" - # Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: # Unique id for this node (autogenerated if empty) @@ -521,7 +516,7 @@ swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/noauth.*}" - title: "${SWAGGER_TITLE:Thingsboard REST API}" + title: "${SWAGGER_TITLE:ThingsBoard REST API}" description: "${SWAGGER_DESCRIPTION:For instructions how to authorize requests please visit REST API documentation page.}" contact: name: "${SWAGGER_CONTACT_NAME:Thingsboard team}" @@ -568,4 +563,7 @@ queue: topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" service: - type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport \ No newline at end of file + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/common/queue/pom.xml b/common/queue/pom.xml index fd26358cc6..429f68acb0 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -40,6 +40,10 @@ org.thingsboard.common data + + org.thingsboard.common + util + org.thingsboard.common message @@ -84,6 +88,14 @@ ch.qos.logback logback-classic + + com.google.protobuf + protobuf-java + + + org.apache.curator + curator-recipes + junit junit @@ -94,10 +106,6 @@ mockito-all test - - com.google.protobuf - protobuf-java - diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java index 7f8cff5f22..6917c23719 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java index 3d7d791ae4..e823d2a5fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueCallback { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index ca51eb4ddb..ddf9d7d9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java index 22714af77e..ca8f3a61da 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java index 9dad87588d..f95454c976 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java index 8eecae51fb..a3331999a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueMsgMetadata { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 41d165d743..2ec01357b7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -1,6 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.discovery.TopicPartitionInfo; public interface TbQueueProducer { @@ -12,4 +28,6 @@ public interface TbQueueProducer { ListenableFuture send(String topic, T msg, TbQueueCallback callback); + ListenableFuture send(String topic, int partition, T msg, TbQueueCallback callback); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 5ba28f22c5..5182bb7260 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java index f3b4e4ad5d..686570ca32 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server; public interface TbQueueResponseTemplate { diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java index 2f406a9631..118cf5e0b7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import java.nio.ByteBuffer; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java index e547a6c070..de0c368a65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import org.thingsboard.server.TbQueueMsgHeaders; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 49022ad2e3..6876c2f3d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 92dac0551b..ba32b399ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Builder; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java index c072dc4d67..8dd492504c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common; import lombok.Data; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java new file mode 100644 index 0000000000..0b09a333d8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { + + @Getter + @Value("${service.id:#{null}}") + private String serviceId; + + @Getter + @Value("${service.type:monolith}") + private String serviceType; + + @Getter + @Value("${service.tenant_id:}") + private String tenantIdStr; + + private List serviceTypes; + private ServiceInfo serviceInfo; + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(serviceId)) { + try { + serviceId = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + serviceId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); + } + } + log.info("Current Service ID: {}", serviceId); + if (serviceType.equalsIgnoreCase("monolith")) { + serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); + } else { + serviceTypes = Collections.singletonList(ServiceType.valueOf(serviceType)); + } + ServiceInfo.Builder builder = ServiceInfo.newBuilder() + .setServiceId(serviceId) + .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + if (!StringUtils.isEmpty(tenantIdStr)) { + UUID tenantId = UUID.fromString(tenantIdStr); + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + } + serviceInfo = builder.build(); + } + + + @Override + public List getSupportedServiceTypes() { + return serviceTypes; + } + + @Override + public ServiceInfo getServiceInfo() { + return serviceInfo; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java new file mode 100644 index 0000000000..7d083ba873 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + + +public class PartitionChangeEvent extends ApplicationEvent { + + @Getter + private final List partitions; + + public PartitionChangeEvent(Object source, List partitions) { + super(source); + this.partitions = partitions; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java new file mode 100644 index 0000000000..d85b1d3f44 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; + +public interface PartitionDiscoveryService { + + List getCurrentPartitions(ServiceType serviceType); + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java new file mode 100644 index 0000000000..801069eb61 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +public enum ServiceType { + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java new file mode 100644 index 0000000000..5cb545c5d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import java.util.List; + +public interface TbServiceInfoProvider { + + List getSupportedServiceTypes(); + + String getServiceId(); + + ServiceInfo getServiceInfo(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java new file mode 100644 index 0000000000..b46b941360 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Data; + +@Data +public class TopicPartitionInfo { + + private String topic; + private int partition; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java new file mode 100644 index 0000000000..a722f2f395 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -0,0 +1,255 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.framework.state.ConnectionStateListener; +import org.apache.curator.retry.RetryForever; +import org.apache.curator.utils.CloseableUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cluster.ServerAddress; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) +@Slf4j +public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, PathChildrenCacheListener { + + @Value("${zk.url}") + private String zkUrl; + @Value("${zk.retry_interval_ms}") + private Integer zkRetryInterval; + @Value("${zk.connection_timeout_ms}") + private Integer zkConnectionTimeout; + @Value("${zk.session_timeout_ms}") + private Integer zkSessionTimeout; + @Value("${zk.zk_dir}") + private String zkDir; + + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.rule_engine.partitions:100}") + private Integer ruleEnginePartitions; + + @Autowired + private TbServiceInfoProvider serviceIdProvider; + + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private final ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + + private ExecutorService reconnectExecutorService; + private CuratorFramework client; + private PathChildrenCache cache; + private String nodePath; + private String zkNodesDir; + + private volatile boolean stopped = true; + + @Override + public List getCurrentPartitions(ServiceType serviceType) { + return Collections.emptyList(); + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + + } + + @PostConstruct + public void init() { + log.info("Initializing..."); + Assert.hasLength(zkUrl, missingProperty("zk.url")); + Assert.notNull(zkRetryInterval, missingProperty("zk.retry_interval_ms")); + Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); + Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); + + reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); + + partitionSizes.put(ServiceType.TB_CORE, corePartitions); + partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); + + log.info("Initializing discovery service using ZK connect string: {}", zkUrl); + + zkNodesDir = zkDir + "/nodes"; + initZkClient(); + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + if (stopped) { + log.debug("Ignoring application ready event. Service is stopped."); + return; + } else { + log.info("Received application ready event. Starting current ZK node."); + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); + return; + } + publishCurrentServer(); + getOtherServers().forEach( + server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) + ); + } + + @Override + public synchronized void publishCurrentServer() { + ServerInstance self = this.serverInstance.getSelf(); + if (currentServerExists()) { + log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + } else { + try { + log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + nodePath = client.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); + log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + client.getConnectionStateListenable().addListener(checkReconnect(self)); + } catch (Exception e) { + log.error("Failed to create ZK node", e); + throw new RuntimeException(e); + } + } + } + + private boolean currentServerExists() { + if (nodePath == null) { + return false; + } + try { + ServerInstance self = this.serverInstance.getSelf(); + ServerAddress registeredServerAdress = null; + registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); + if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + return true; + } + } catch (KeeperException.NoNodeException e) { + log.info("ZK node does not exist: {}", nodePath); + } catch (Exception e) { + log.error("Couldn't check if ZK node exists", e); + } + return false; + } + + private ConnectionStateListener checkReconnect(ServerInstance self) { + return (client, newState) -> { + log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + if (newState == ConnectionState.LOST) { + reconnectExecutorService.submit(this::reconnect); + } + }; + } + + private volatile boolean reconnectInProgress = false; + + private synchronized void reconnect() { + if (!reconnectInProgress) { + reconnectInProgress = true; + try { + destroyZkClient(); + initZkClient(); + publishCurrentServer(); + } catch (Exception e) { + log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); + } finally { + reconnectInProgress = false; + } + } + } + + private void initZkClient() { + try { + client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); + client.start(); + client.blockUntilConnected(); + cache = new PathChildrenCache(client, zkNodesDir, true); + cache.getListenable().addListener(this); + cache.start(); + stopped = false; + log.info("ZK client connected"); + } catch (Exception e) { + log.error("Failed to connect to ZK: {}", e.getMessage(), e); + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + throw new RuntimeException(e); + } + } + + private void unpublishCurrentServer() { + try { + if (nodePath != null) { + client.delete().forPath(nodePath); + } + } catch (Exception e) { + log.error("Failed to delete ZK node {}", nodePath, e); + throw new RuntimeException(e); + } + } + + private void destroyZkClient() { + stopped = true; + try { + unpublishCurrentServer(); + } catch (Exception e) { + } + CloseableUtils.closeQuietly(cache); + CloseableUtils.closeQuietly(client); + log.info("ZK client disconnected"); + } + + @PreDestroy + public void destroy() { + destroyZkClient(); + reconnectExecutorService.shutdownNow(); + log.info("Stopped discovery service"); + } + + public static String missingProperty(String propertyName) { + return "The " + propertyName + " property need to be set!"; + } + + @Override + public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { + + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java index e44a93a70d..9f8e73bdfc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import org.apache.kafka.clients.consumer.ConsumerRecord; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java index 4698c226d6..09cc292deb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.kafka; import lombok.AllArgsConstructor; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java index abf30667b5..ded4cd9810 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java index a0a35eefb8..b8ddcff89c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 32b9399164..c16d34a81a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.memory; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 2f9a9ad8f8..62bc2dd5ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 945888a45a..265373793a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index b7e210f0f6..43878687ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java index 14d8efd534..de69e1819a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.provider; import org.thingsboard.server.TbQueueConsumer; diff --git a/common/queue/src/main/proto/transport.proto b/common/queue/src/main/proto/transport.proto index 892139a84a..8c564b6540 100644 --- a/common/queue/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/transport.proto @@ -19,6 +19,16 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +/** + * Service Discovery Data Structures; + */ +message ServiceInfo { + string serviceId = 1; + repeated string serviceTypes = 2; + int64 tenantIdMSB = 3; + int64 tenantIdLSB = 4; +} + /** * Transport Service Data Structures; */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index e4dc2ba1f1..cdf6b3f763 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 23c017567bf59022bef311c9b2d641fc3c31c7a4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 16 Mar 2020 19:24:56 +0200 Subject: [PATCH 012/196] Implementation of Queue Interfaces --- .../server/actors/ActorSystemContext.java | 21 +- .../server/actors/app/AppActor.java | 14 +- .../device/DeviceActorMessageProcessor.java | 2 +- .../actors/rpc/BasicRpcSessionListener.java | 83 ----- .../server/actors/rpc/RpcBroadcastMsg.java | 27 -- .../server/actors/rpc/RpcManagerActor.java | 230 ------------ .../server/actors/rpc/RpcSessionActor.java | 135 ------- .../actors/rpc/RpcSessionClosedMsg.java | 29 -- .../actors/rpc/RpcSessionConnectedMsg.java | 31 -- .../rpc/RpcSessionCreateRequestMsg.java | 35 -- .../actors/rpc/RpcSessionDisconnectedMsg.java | 29 -- .../server/actors/rpc/RpcSessionTellMsg.java | 27 -- .../server/actors/rpc/SessionActorInfo.java | 30 -- .../actors/ruleChain/DefaultTbContext.java | 21 +- .../RuleChainActorMessageProcessor.java | 15 +- .../server/actors/service/ActorService.java | 4 +- .../actors/service/DefaultActorService.java | 213 +++++------ .../server/actors/stats/StatsActor.java | 3 +- .../CurrentServerInstanceService.java | 55 --- .../cluster/discovery/DiscoveryService.java | 33 -- .../discovery/DiscoveryServiceListener.java | 28 -- .../discovery/DummyDiscoveryService.java | 67 ---- .../cluster/discovery/ServerInstance.java | 47 --- .../discovery/ServerInstanceService.java | 24 -- .../cluster/discovery/ZkDiscoveryService.java | 330 ------------------ .../routing/ClusterRoutingService.java | 35 -- .../ConsistentClusterRoutingService.java | 153 -------- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 40 ++- .../queue/TbRuleEngineConsumerStats.java | 59 +--- .../service/rpc/DefaultDeviceRpcService.java | 29 +- .../state/DefaultDeviceStateService.java | 91 ++--- .../DefaultTelemetrySubscriptionService.java | 38 +- .../BaseRuleChainTransactionService.java | 25 +- .../DefaultTbCoreToTransportService.java | 9 +- .../RemoteRuleEngineTransportService.java | 237 ------------- .../transport/RemoteTransportApiService.java | 3 +- .../transport/TransportApiRequestDecoder.java | 31 -- .../TransportApiResponseEncoder.java | 30 -- .../src/main/resources/thingsboard.yml | 3 +- .../ConsistentClusterRoutingServiceTest.java | 71 ++-- .../thingsboard/server/TbQueueProducer.java | 10 +- .../common/DefaultTbQueueRequestTemplate.java | 5 +- .../DefaultTbQueueResponseTemplate.java | 3 +- .../discovery}/ConsistentHashCircle.java | 18 +- .../ConsistentHashPartitionService.java | 221 ++++++++++++ .../DefaultTbServiceInfoProvider.java | 11 +- .../discovery/DummyDiscoveryService.java | 32 ++ .../discovery/PartitionDiscoveryService.java | 9 - .../server/discovery/PartitionService.java | 16 + .../server/discovery/TopicPartitionInfo.java | 23 +- .../ZkPartitionDiscoveryService.java | 122 ++++--- .../environment/EnvironmentLogService.java | 4 +- .../server/kafka/TBKafkaProducerTemplate.java | 50 +-- .../memory/InMemoryTbQueueProducer.java | 17 +- ...ava => InMemoryMonolithQueueProvider.java} | 12 +- .../provider/KafkaMonolithQueueProvider.java | 119 +++++++ .../provider/KafkaTbCoreQueueProvider.java | 33 +- .../KafkaTbRuleEngineQueueProvider.java | 86 +++++ .../provider/KafkaTransportQueueProvider.java | 8 +- .../provider/TbRuleEngineQueueProvider.java | 61 ++++ common/transport/transport-api/pom.xml | 4 + .../service/DefaultTransportService.java | 38 +- 63 files changed, 1055 insertions(+), 2236 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java rename {application/src/main/java/org/thingsboard/server/service/cluster/routing => common/queue/src/main/java/org/thingsboard/server/discovery}/ConsistentHashCircle.java (66%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java rename {application/src/main/java/org/thingsboard/server/service => common/queue/src/main/java/org/thingsboard/server}/environment/EnvironmentLogService.java (90%) rename common/queue/src/main/java/org/thingsboard/server/provider/{InMemoryTbCoreQueueProvider.java => InMemoryMonolithQueueProvider.java} (85%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index d10436075b..e5a0e6a25a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -65,8 +65,6 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -108,19 +106,11 @@ public class ActorSystemContext { @Setter private ActorService actorService; - @Autowired - @Getter - private DiscoveryService discoveryService; - @Autowired @Getter @Setter private ComponentDiscoveryService componentService; - @Autowired - @Getter - private ClusterRoutingService routingService; - @Autowired @Getter private DataDecodingEncodingService encodingService; @@ -368,7 +358,8 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + //TODO 2.5 +// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); persistEvent(event); } @@ -377,7 +368,8 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + //TODO 2.5 +// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -406,8 +398,11 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); } + public String getServerAddress() { - return discoveryService.getCurrentServer().getServerAddress().toString(); + //TODO 2.5 +// return discoveryService.getCurrentServer().getServerAddress().toString(); + return null; } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 778c945a32..06f936a9e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -134,13 +134,15 @@ public class AppActor extends RuleChainManagerActor { } private void onPossibleClusterMsg(SendToClusterMsg msg) { - Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); - if (address.isPresent()) { - systemContext.getRpcService().tell( - systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); - } else { + //TODO 2.5 +// Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); +// if (address.isPresent()) { + +// systemContext.getRpcService().tell( +// systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); +// } else { self().tell(msg.getMsg(), ActorRef.noSender()); - } +// } } private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6c0dde4044..6b878ef7d8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -260,7 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - //TODO: 2.5 move this as a notification to the queue; + //TODO 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java b/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java deleted file mode 100644 index 8b502b973c..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/BasicRpcSessionListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class BasicRpcSessionListener implements GrpcSessionListener { - - private final ClusterRpcCallbackExecutorService callbackExecutorService; - private final ActorService service; - private final ActorRef manager; - private final ActorRef self; - - BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) { - this.service = context.getActorService(); - this.callbackExecutorService = context.getClusterRpcCallbackExecutor(); - this.manager = manager; - this.self = self; - } - - @Override - public void onConnected(GrpcSession session) { - log.info("[{}][{}] session started", session.getRemoteServer(), getType(session)); - if (!session.isClient()) { - manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self); - } - } - - @Override - public void onDisconnected(GrpcSession session) { - log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session)); - manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self); - } - - @Override - public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) { - log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage); - callbackExecutorService.execute(() -> { - try { - service.onReceivedMsg(session.getRemoteServer(), clusterMessage); - } catch (Exception e) { - log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e); - } - }); - } - - @Override - public void onError(GrpcSession session, Throwable t) { - log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t); - manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self); - session.close(); - } - - private static String getType(GrpcSession session) { - return session.isClient() ? "Client" : "Server"; - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java deleted file mode 100644 index f0ff8ce9e1..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcBroadcastMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcBroadcastMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java deleted file mode 100644 index 133e7375d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcManagerActor.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.actor.Props; -import akka.actor.SupervisorStrategy; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import scala.concurrent.duration.Duration; - -import java.util.*; - -/** - * @author Andrew Shvayka - */ -public class RpcManagerActor extends ContextAwareActor { - - private final Map sessionActors; - private final Map> pendingMsgs; - private final ServerAddress instance; - - private RpcManagerActor(ActorSystemContext systemContext) { - super(systemContext); - this.sessionActors = new HashMap<>(); - this.pendingMsgs = new HashMap<>(); - this.instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - - systemContext.getDiscoveryService().getOtherServers().stream() - .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0) - .forEach(otherServer -> onCreateSessionRequest( - new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null))); - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - onMsg((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcBroadcastMsg) { - onMsg((RpcBroadcastMsg) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - onCreateSessionRequest((RpcSessionCreateRequestMsg) msg); - } else if (msg instanceof RpcSessionConnectedMsg) { - onSessionConnected((RpcSessionConnectedMsg) msg); - } else if (msg instanceof RpcSessionDisconnectedMsg) { - onSessionDisconnected((RpcSessionDisconnectedMsg) msg); - } else if (msg instanceof RpcSessionClosedMsg) { - onSessionClosed((RpcSessionClosedMsg) msg); - } else if (msg instanceof ClusterEventMsg) { - onClusterEvent((ClusterEventMsg) msg); - } - } - - private void onMsg(RpcBroadcastMsg msg) { - log.debug("Forwarding msg to session actors {}", msg); - sessionActors.keySet().forEach(address -> { - ClusterAPIProtos.ClusterMessage msgWithServerAddress = msg.getMsg() - .toBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(address.getHost()) - .setPort(address.getPort()) - .build()) - .build(); - onMsg(msgWithServerAddress); - }); - pendingMsgs.values().forEach(queue -> queue.add(msg.getMsg())); - } - - private void onMsg(ClusterAPIProtos.ClusterMessage msg) { - if (msg.hasServerAddress()) { - ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE); - SessionActorInfo session = sessionActors.get(address); - if (session != null) { - log.debug("{} Forwarding msg to session actor: {}", address, msg); - session.getActor().tell(msg, ActorRef.noSender()); - } else { - log.debug("{} Storing msg to pending queue: {}", address, msg); - Queue queue = pendingMsgs.get(address); - if (queue == null) { - queue = new LinkedList<>(); - pendingMsgs.put(new ServerAddress( - msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE), queue); - } - queue.add(msg); - } - } else { - log.warn("Cluster msg doesn't have server address [{}]", msg); - } - } - - @Override - public void postStop() { - sessionActors.clear(); - pendingMsgs.clear(); - } - - private void onClusterEvent(ClusterEventMsg msg) { - ServerAddress server = msg.getServerAddress(); - if (server.compareTo(instance) > 0) { - if (msg.isAdded()) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(UUID.randomUUID(), server, null)); - } else { - onSessionClose(false, server); - } - } - } - - private void onSessionConnected(RpcSessionConnectedMsg msg) { - register(msg.getRemoteAddress(), msg.getId(), context().sender()); - } - - private void onSessionDisconnected(RpcSessionDisconnectedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private void onSessionClosed(RpcSessionClosedMsg msg) { - boolean reconnect = msg.isClient() && isRegistered(msg.getRemoteAddress()); - onSessionClose(reconnect, msg.getRemoteAddress()); - } - - private boolean isRegistered(ServerAddress address) { - for (ServerInstance server : systemContext.getDiscoveryService().getOtherServers()) { - if (server.getServerAddress().equals(address)) { - return true; - } - } - return false; - } - - private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) { - log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect); - SessionActorInfo sessionRef = sessionActors.get(remoteAddress); - if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) { - context().stop(sessionRef.actor); - sessionActors.remove(remoteAddress); - pendingMsgs.remove(remoteAddress); - if (reconnect) { - onCreateSessionRequest(new RpcSessionCreateRequestMsg(sessionRef.sessionId, remoteAddress, null)); - } - } - } - - private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) { - if (msg.getRemoteAddress() != null) { - if (!sessionActors.containsKey(msg.getRemoteAddress())) { - ActorRef actorRef = createSessionActor(msg); - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef); - } - } else { - createSessionActor(msg); - } - } - - private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) { - sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender)); - log.info("[{}][{}] Registering session actor.", remoteAddress, uuid); - Queue data = pendingMsgs.remove(remoteAddress); - if (data != null) { - log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size()); - data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender())); - } else { - log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid); - } - } - - private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Creating session actor.", msg.getMsgUid()); - ActorRef actor = context().actorOf( - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())) - .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME)); - actor.tell(msg, context().self()); - return actor; - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - public ActorCreator(ActorSystemContext context) { - super(context); - } - - @Override - public RpcManagerActor create() { - return new RpcManagerActor(context); - } - } - - @Override - public SupervisorStrategy supervisorStrategy() { - return strategy; - } - - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { - log.warn("Unknown failure", t); - return SupervisorStrategy.resume(); - }); -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java deleted file mode 100644 index af2a7f0631..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionActor.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.actors.service.ContextBasedCreator; -import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.gen.cluster.ClusterRpcServiceGrpc; -import org.thingsboard.server.service.cluster.rpc.GrpcSession; -import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener; - -import java.util.UUID; - -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CONNECT_RPC_MESSAGE; - -/** - * @author Andrew Shvayka - */ -@Slf4j -public class RpcSessionActor extends ContextAwareActor { - - - private final UUID sessionId; - private GrpcSession session; - private GrpcSessionListener listener; - - private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) { - super(systemContext); - this.sessionId = sessionId; - } - - @Override - protected boolean process(TbActorMsg msg) { - //TODO Move everything here, to work with TbActorMsg - return false; - } - - @Override - public void onReceive(Object msg) { - if (msg instanceof ClusterAPIProtos.ClusterMessage) { - tell((ClusterAPIProtos.ClusterMessage) msg); - } else if (msg instanceof RpcSessionCreateRequestMsg) { - initSession((RpcSessionCreateRequestMsg) msg); - } - } - - private void tell(ClusterAPIProtos.ClusterMessage msg) { - if (session != null) { - session.sendMsg(msg); - } else { - log.trace("Failed to send message due to missing session!"); - } - } - - @Override - public void postStop() { - if (session != null) { - log.info("Closing session -> {}", session.getRemoteServer()); - try { - session.close(); - } catch (RuntimeException e) { - log.trace("Failed to close session!", e); - } - } - } - - private void initSession(RpcSessionCreateRequestMsg msg) { - log.info("[{}] Initializing session", context().self()); - ServerAddress remoteServer = msg.getRemoteAddress(); - listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self()); - if (msg.getRemoteAddress() == null) { - // Server session - session = new GrpcSession(listener); - session.setOutputStream(msg.getResponseObserver()); - session.initInputStream(); - session.initOutputStream(); - systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream()); - } else { - // Client session - ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build(); - session = new GrpcSession(remoteServer, listener, channel); - session.initInputStream(); - - ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel); - StreamObserver outputStream = stub.handleMsgs(session.getInputStream()); - - session.setOutputStream(outputStream); - session.initOutputStream(); - outputStream.onNext(toConnectMsg()); - } - } - - public static class ActorCreator extends ContextBasedCreator { - private static final long serialVersionUID = 1L; - - private final UUID sessionId; - - public ActorCreator(ActorSystemContext context, UUID sessionId) { - super(context); - this.sessionId = sessionId; - } - - @Override - public RpcSessionActor create() { - return new RpcSessionActor(context, sessionId); - } - } - - private ClusterAPIProtos.ClusterMessage toConnectMsg() { - ServerAddress instance = systemContext.getDiscoveryService().getCurrentServer().getServerAddress(); - return ClusterAPIProtos.ClusterMessage.newBuilder().setMessageType(CONNECT_RPC_MESSAGE).setServerAddress( - ClusterAPIProtos.ServerAddress.newBuilder().setHost(instance.getHost()) - .setPort(instance.getPort()).build()).build(); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java deleted file mode 100644 index bb87d2ea88..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionClosedMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionClosedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java deleted file mode 100644 index 489a5bfa17..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionConnectedMsg.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionConnectedMsg { - - private final ServerAddress remoteAddress; - private final UUID id; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java deleted file mode 100644 index df41a7e95f..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionCreateRequestMsg.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import io.grpc.stub.StreamObserver; -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionCreateRequestMsg { - - private final UUID msgUid; - private final ServerAddress remoteAddress; - private final StreamObserver responseObserver; - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java deleted file mode 100644 index 14e7504637..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionDisconnectedMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionDisconnectedMsg { - - private final boolean client; - private final ServerAddress remoteAddress; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java b/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java deleted file mode 100644 index 3832d6eb94..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/RpcSessionTellMsg.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import lombok.Data; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -/** - * @author Andrew Shvayka - */ -@Data -public final class RpcSessionTellMsg { - private final ClusterAPIProtos.ClusterMessage msg; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java b/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java deleted file mode 100644 index 811713819a..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/rpc/SessionActorInfo.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.rpc; - -import akka.actor.ActorRef; -import lombok.Data; - -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -@Data -public final class SessionActorInfo { - protected final UUID sessionId; - protected final ActorRef actor; -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b700007498..f5ea5c399f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -121,8 +121,10 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - Optional address = mainCtx.getRoutingService().resolveById(entityId); - return !address.isPresent(); + //TODO 2.5 +// Optional address = mainCtx.getRoutingService().resolveById(entityId); +// return !address.isPresent(); + return true; } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -353,13 +355,14 @@ class DefaultTbContext implements TbContext { src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { if (src.isRestApiCall()) { - ServerAddress requestOriginAddress; - if (!StringUtils.isEmpty(src.getOriginHost())) { - requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); - } else { - requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); - } - mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); + //TODO 2.5 +// ServerAddress requestOriginAddress; +// if (!StringUtils.isEmpty(src.getOriginHost())) { +// requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); +// } else { +// requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); +// } +// mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); } consumer.accept(RuleEngineDeviceRpcResponse.builder() .deviceId(src.getDeviceId()) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 49d019c1dc..095af7f62d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -230,20 +230,21 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); - - if (address.isPresent()) { - onRemoteTellNext(address.get(), envelope); - } else { + //TODO 2.5 +// Optional address = systemContext.getRoutingService().resolveById(originatorEntityId); +// if (address.isPresent()) { +// onRemoteTellNext(address.get(), envelope); +// } else { onLocalTellNext(envelope); - } +// } } private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { TbMsg msg = envelope.getMsg(); log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + //TODO 2.5 +// systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); } private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 48a31127f4..890128e645 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -21,10 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.rpc.RpcMsgListener; -public interface ActorService extends SessionMsgProcessor, RpcMsgListener, DiscoveryServiceListener { +public interface ActorService extends SessionMsgProcessor { void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index e5da93b9e2..fc9eb537c1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -19,7 +19,6 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; import akka.actor.Props; import akka.actor.Terminated; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -32,25 +31,16 @@ import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.app.AppActor; import org.thingsboard.server.actors.app.AppInitMsg; -import org.thingsboard.server.actors.rpc.RpcBroadcastMsg; -import org.thingsboard.server.actors.rpc.RpcManagerActor; -import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg; import org.thingsboard.server.actors.stats.StatsActor; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DeviceStateService; import scala.concurrent.Await; import scala.concurrent.Future; @@ -60,8 +50,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - @Service @Slf4j public class DefaultActorService implements ActorService { @@ -77,12 +65,6 @@ public class DefaultActorService implements ActorService { @Autowired private ActorSystemContext actorContext; - @Autowired - private ClusterRpcService rpcService; - - @Autowired - private DiscoveryService discoveryService; - @Autowired private DeviceStateService deviceStateService; @@ -102,13 +84,9 @@ public class DefaultActorService implements ActorService { appActor = system.actorOf(Props.create(new AppActor.ActorCreator(actorContext)).withDispatcher(APP_DISPATCHER_NAME), "appActor"); actorContext.setAppActor(appActor); - rpcManagerActor = system.actorOf(Props.create(new RpcManagerActor.ActorCreator(actorContext)).withDispatcher(CORE_DISPATCHER_NAME), - "rpcManagerActor"); - ActorRef statsActor = system.actorOf(Props.create(new StatsActor.ActorCreator(actorContext)).withDispatcher(CORE_DISPATCHER_NAME), "statsActor"); actorContext.setStatsActor(statsActor); - rpcService.init(this); log.info("Actor system initialized."); } @@ -134,22 +112,23 @@ public class DefaultActorService implements ActorService { appActor.tell(msg, ActorRef.noSender()); } - @Override - public void onServerAdded(ServerInstance server) { - log.trace("Processing onServerAdded msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), true)); - } - - @Override - public void onServerUpdated(ServerInstance server) { - //Do nothing - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.trace("Processing onServerRemoved msg: {}", server); - broadcast(new ClusterEventMsg(server.getServerAddress(), false)); - } + //TODO 2.5 +// @Override +// public void onServerAdded(ServerInstance server) { +// log.trace("Processing onServerAdded msg: {}", server); +// broadcast(new ClusterEventMsg(server.getServerAddress(), true)); +// } +// +// @Override +// public void onServerUpdated(ServerInstance server) { +// //Do nothing +// } +// +// @Override +// public void onServerRemoved(ServerInstance server) { +// log.trace("Processing onServerRemoved msg: {}", server); +// broadcast(new ClusterEventMsg(server.getServerAddress(), false)); +// } @Override public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { @@ -172,12 +151,13 @@ public class DefaultActorService implements ActorService { public void broadcast(ToAllNodesMsg msg) { actorContext.getEncodingService().encode(msg); - rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage - .newBuilder() - .setPayload(ByteString - .copyFrom(actorContext.getEncodingService().encode(msg))) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .build())); + //TODO 2.5 +// rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage +// .newBuilder() +// .setPayload(ByteString +// .copyFrom(actorContext.getEncodingService().encode(msg))) +// .setMessageType(CLUSTER_ACTOR_MESSAGE) +// .build())); appActor.tell(msg, ActorRef.noSender()); } @@ -204,79 +184,78 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - receivedClusterMsgs.incrementAndGet(); - } - ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); - if (log.isDebugEnabled()) { - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); - log.info("MSG: {}", msg); - } - switch (msg.getMessageType()) { - case CLUSTER_ACTOR_MESSAGE: - java.util.Optional decodedMsg = actorContext.getEncodingService() - .decode(msg.getPayload().toByteArray()); - if (decodedMsg.isPresent()) { - appActor.tell(decodedMsg.get(), ActorRef.noSender()); - } else { - log.error("Error during decoding cluster proto message"); - } - break; - case TO_ALL_NODES_MSG: - //TODO - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: - actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: - actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: - actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: - actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: - actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); - break; - case CLUSTER_TRANSACTION_SERVICE_MESSAGE: - actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); - break; - } - } - - @Override - public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { - if (statsEnabled) { - sentClusterMsgs.incrementAndGet(); - } - rpcManagerActor.tell(msg, ActorRef.noSender()); - } - - @Override - public void onBroadcastMsg(RpcBroadcastMsg msg) { - rpcManagerActor.tell(msg, ActorRef.noSender()); - } + //TODO 2.5 +// @Override +// public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { +// if (statsEnabled) { +// receivedClusterMsgs.incrementAndGet(); +// } +// ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); +// if (log.isDebugEnabled()) { +// log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); +// log.info("MSG: {}", msg); +// } +// switch (msg.getMessageType()) { +// case CLUSTER_ACTOR_MESSAGE: +// java.util.Optional decodedMsg = actorContext.getEncodingService() +// .decode(msg.getPayload().toByteArray()); +// if (decodedMsg.isPresent()) { +// appActor.tell(decodedMsg.get(), ActorRef.noSender()); +// } else { +// log.error("Error during decoding cluster proto message"); +// } +// break; +// case TO_ALL_NODES_MSG: +// //TODO +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: +// actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: +// actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: +// actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: +// actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: +// actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: +// actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); +// break; +// case CLUSTER_TRANSACTION_SERVICE_MESSAGE: +// actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); +// break; +// } +// } +// @Override +// public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { +// if (statsEnabled) { +// sentClusterMsgs.incrementAndGet(); +// } +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } +// +// @Override +// public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { +// if (statsEnabled) { +// sentClusterMsgs.incrementAndGet(); +// } +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } +// @Override +// public void onBroadcastMsg(RpcBroadcastMsg msg) { +// rpcManagerActor.tell(msg, ActorRef.noSender()); +// } @Override public void onDeviceAdded(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 31461a8fac..c8e55bf9ed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -58,7 +58,8 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + //TODO 2.5 +// event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java deleted file mode 100644 index 9a7c83f167..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/CurrentServerInstanceService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; - -import javax.annotation.PostConstruct; - -import static org.thingsboard.server.utils.MiscUtils.missingProperty; - -/** - * @author Andrew Shvayka - */ -@Service -@Slf4j -public class CurrentServerInstanceService implements ServerInstanceService { - - @Value("${rpc.bind_host}") - private String rpcHost; - @Value("${rpc.bind_port}") - private Integer rpcPort; - - private ServerInstance self; - - @PostConstruct - public void init() { - Assert.hasLength(rpcHost, missingProperty("rpc.bind_host")); - Assert.notNull(rpcPort, missingProperty("rpc.bind_port")); - self = new ServerInstance(new ServerAddress(rpcHost, rpcPort, ServerType.CORE)); - log.info("Current server instance: [{};{}]", self.getHost(), self.getPort()); - } - - @Override - public ServerInstance getSelf() { - return self; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java deleted file mode 100644 index d0ba27814c..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryService.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import java.util.List; - -/** - * @author Andrew Shvayka - */ -public interface DiscoveryService { - - void publishCurrentServer(); - - void unpublishCurrentServer(); - - ServerInstance getCurrentServer(); - - List getOtherServers(); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java deleted file mode 100644 index ef6b565817..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DiscoveryServiceListener.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -/** - * @author Andrew Shvayka - */ -public interface DiscoveryServiceListener { - - void onServerAdded(ServerInstance server); - - void onServerUpdated(ServerInstance server); - - void onServerRemoved(ServerInstance server); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java deleted file mode 100644 index 009a7801e7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/DummyDiscoveryService.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.DependsOn; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.util.Collections; -import java.util.List; - -/** - * @author Andrew Shvayka - */ -@Service -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) -@Slf4j -@DependsOn("environmentLogService") -public class DummyDiscoveryService implements DiscoveryService { - - @Autowired - private ServerInstanceService serverInstance; - - @PostConstruct - public void init() { - log.info("Initializing..."); - } - - @Override - public void publishCurrentServer() { - //Do nothing - } - - @Override - public void unpublishCurrentServer() { - //Do nothing - } - - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); - } - - @Override - public List getOtherServers() { - return Collections.emptyList(); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java deleted file mode 100644 index e9325800af..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstance.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.thingsboard.server.common.msg.cluster.ServerAddress; - -/** - * @author Andrew Shvayka - */ -@ToString -@EqualsAndHashCode(exclude = {"serverInfo", "serverAddress"}) -public final class ServerInstance implements Comparable { - - @Getter - private final String host; - @Getter - private final int port; - @Getter - private final ServerAddress serverAddress; - - public ServerInstance(ServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.host = serverAddress.getHost(); - this.port = serverAddress.getPort(); - } - - @Override - public int compareTo(ServerInstance o) { - return this.serverAddress.compareTo(o.serverAddress); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java deleted file mode 100644 index 095e10f0b3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ServerInstanceService.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -/** - * @author Andrew Shvayka - */ -public interface ServerInstanceService { - - ServerInstance getSelf(); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java deleted file mode 100644 index e1cdd5f83e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/discovery/ZkDiscoveryService.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.discovery; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.SerializationException; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.RetryForever; -import org.apache.curator.utils.CloseableUtils; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; - -/** - * @author Andrew Shvayka - */ -@Service -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) -@Slf4j -public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener { - - @Value("${zk.url}") - private String zkUrl; - @Value("${zk.retry_interval_ms}") - private Integer zkRetryInterval; - @Value("${zk.connection_timeout_ms}") - private Integer zkConnectionTimeout; - @Value("${zk.session_timeout_ms}") - private Integer zkSessionTimeout; - @Value("${zk.zk_dir}") - private String zkDir; - - private String zkNodesDir; - - @Autowired - private ServerInstanceService serverInstance; - - @Autowired - @Lazy - private TelemetrySubscriptionService tsSubService; - - @Autowired - @Lazy - private DeviceStateService deviceStateService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - @Lazy - private ClusterRoutingService routingService; - - private ExecutorService reconnectExecutorService; - - private CuratorFramework client; - private PathChildrenCache cache; - private String nodePath; - - private volatile boolean stopped = true; - - @PostConstruct - public void init() { - log.info("Initializing..."); - Assert.hasLength(zkUrl, MiscUtils.missingProperty("zk.url")); - Assert.notNull(zkRetryInterval, MiscUtils.missingProperty("zk.retry_interval_ms")); - Assert.notNull(zkConnectionTimeout, MiscUtils.missingProperty("zk.connection_timeout_ms")); - Assert.notNull(zkSessionTimeout, MiscUtils.missingProperty("zk.session_timeout_ms")); - - reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); - - log.info("Initializing discovery service using ZK connect string: {}", zkUrl); - - zkNodesDir = zkDir + "/nodes"; - initZkClient(); - } - - private void initZkClient() { - try { - client = CuratorFrameworkFactory.newClient(zkUrl, zkSessionTimeout, zkConnectionTimeout, new RetryForever(zkRetryInterval)); - client.start(); - client.blockUntilConnected(); - cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); - cache.start(); - stopped = false; - log.info("ZK client connected"); - } catch (Exception e) { - log.error("Failed to connect to ZK: {}", e.getMessage(), e); - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - throw new RuntimeException(e); - } - } - - private void destroyZkClient() { - stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) {} - CloseableUtils.closeQuietly(cache); - CloseableUtils.closeQuietly(client); - log.info("ZK client disconnected"); - } - - @PreDestroy - public void destroy() { - destroyZkClient(); - reconnectExecutorService.shutdownNow(); - log.info("Stopped discovery service"); - } - - @Override - public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); - if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); - } else { - try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); - nodePath = client.create() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); - client.getConnectionStateListenable().addListener(checkReconnect(self)); - } catch (Exception e) { - log.error("Failed to create ZK node", e); - throw new RuntimeException(e); - } - } - } - - private boolean currentServerExists() { - if (nodePath == null) { - return false; - } - try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { - return true; - } - } catch (KeeperException.NoNodeException e) { - log.info("ZK node does not exist: {}", nodePath); - } catch (Exception e) { - log.error("Couldn't check if ZK node exists", e); - } - return false; - } - - private ConnectionStateListener checkReconnect(ServerInstance self) { - return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); - if (newState == ConnectionState.LOST) { - reconnectExecutorService.submit(this::reconnect); - } - }; - } - - private volatile boolean reconnectInProgress = false; - - private synchronized void reconnect() { - if (!reconnectInProgress) { - reconnectInProgress = true; - try { - destroyZkClient(); - initZkClient(); - publishCurrentServer(); - } catch (Exception e) { - log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); - } finally { - reconnectInProgress = false; - } - } - } - - @Override - public void unpublishCurrentServer() { - try { - if (nodePath != null) { - client.delete().forPath(nodePath); - } - } catch (Exception e) { - log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); - } - } - - @Override - public ServerInstance getCurrentServer() { - return serverInstance.getSelf(); - } - - @Override - public List getOtherServers() { - return cache.getCurrentData().stream() - .filter(cd -> !cd.getPath().equals(nodePath)) - .map(cd -> { - try { - return new ServerInstance((ServerAddress) SerializationUtils.deserialize(cd.getData())); - } catch (NoSuchElementException e) { - log.error("Failed to decode ZK node", e); - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting current ZK node."); - if (stopped) { - log.debug("Ignoring application ready event. Service is stopped."); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); - return; - } - publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); - } - - @Override - public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { - if (stopped) { - log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent); - return; - } - if (client.getState() != CuratorFrameworkState.STARTED) { - log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState()); - return; - } - ChildData data = pathChildrenCacheEvent.getData(); - if (data == null) { - log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent); - return; - } else if (data.getData() == null) { - log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent); - return; - } else if (nodePath != null && nodePath.equals(data.getPath())) { - if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) { - log.info("ZK node for current instance is somehow deleted."); - publishCurrentServer(); - } - log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); - return; - } - ServerInstance instance; - try { - ServerAddress serverAddress = SerializationUtils.deserialize(data.getData()); - instance = new ServerInstance(serverAddress); - } catch (SerializationException e) { - log.error("Failed to decode server instance for node {}", data.getPath(), e); - throw e; - } - log.info("Processing [{}] event for [{}:{}]", pathChildrenCacheEvent.getType(), instance.getHost(), instance.getPort()); - switch (pathChildrenCacheEvent.getType()) { - case CHILD_ADDED: - routingService.onServerAdded(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerAdded(instance); - break; - case CHILD_UPDATED: - routingService.onServerUpdated(instance); - actorService.onServerUpdated(instance); - break; - case CHILD_REMOVED: - routingService.onServerRemoved(instance); - tsSubService.onClusterUpdate(); - deviceStateService.onClusterUpdate(); - actorService.onServerRemoved(instance); - break; - default: - break; - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java deleted file mode 100644 index e0ed64fdbd..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ClusterRoutingService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; - -import java.util.Optional; -import java.util.UUID; - -/** - * @author Andrew Shvayka - */ -public interface ClusterRoutingService extends DiscoveryServiceListener { - - ServerAddress getCurrentServer(); - - Optional resolveById(EntityId entityId); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java b/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java deleted file mode 100644 index e114a9b6ab..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingService.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cluster.routing; - -import com.google.common.hash.HashCode; -import com.google.common.hash.HashFunction; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.Assert; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.DiscoveryServiceListener; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; -import org.thingsboard.server.utils.MiscUtils; - -import javax.annotation.PostConstruct; -import java.util.Arrays; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Cluster service implementation based on consistent hash ring - */ - -@Service -@Slf4j -public class ConsistentClusterRoutingService implements ClusterRoutingService { - - @Autowired - private DiscoveryService discoveryService; - - @Value("${cluster.hash_function_name}") - private String hashFunctionName; - @Value("${cluster.vitrual_nodes_size}") - private Integer virtualNodesSize; - - private ServerInstance currentServer; - - private HashFunction hashFunction; - - private ConsistentHashCircle[] circles; - private ConsistentHashCircle rootCircle; - - @PostConstruct - public void init() { - log.info("Initializing Cluster routing service!"); - this.hashFunction = MiscUtils.forName(hashFunctionName); - this.currentServer = discoveryService.getCurrentServer(); - this.circles = new ConsistentHashCircle[ServerType.values().length]; - for (ServerType serverType : ServerType.values()) { - circles[serverType.ordinal()] = new ConsistentHashCircle(); - } - rootCircle = circles[ServerType.CORE.ordinal()]; - addNode(discoveryService.getCurrentServer()); - for (ServerInstance instance : discoveryService.getOtherServers()) { - addNode(instance); - } - logCircle(); - log.info("Cluster routing service initialized!"); - } - - @Override - public ServerAddress getCurrentServer() { - return discoveryService.getCurrentServer().getServerAddress(); - } - - @Override - public Optional resolveById(EntityId entityId) { - return resolveByUuid(rootCircle, entityId.getId()); - } - - private Optional resolveByUuid(ConsistentHashCircle circle, UUID uuid) { - Assert.notNull(uuid); - if (circle.isEmpty()) { - return Optional.empty(); - } - Long hash = hashFunction.newHasher().putLong(uuid.getMostSignificantBits()) - .putLong(uuid.getLeastSignificantBits()).hash().asLong(); - if (!circle.containsKey(hash)) { - ConcurrentNavigableMap tailMap = - circle.tailMap(hash); - hash = tailMap.isEmpty() ? - circle.firstKey() : tailMap.firstKey(); - } - ServerInstance result = circle.get(hash); - if (!currentServer.equals(result)) { - return Optional.of(result.getServerAddress()); - } else { - return Optional.empty(); - } - } - - @Override - public void onServerAdded(ServerInstance server) { - log.info("On server added event: {}", server); - addNode(server); - logCircle(); - } - - @Override - public void onServerUpdated(ServerInstance server) { - log.debug("Ignoring server onUpdate event: {}", server); - } - - @Override - public void onServerRemoved(ServerInstance server) { - log.info("On server removed event: {}", server); - removeNode(server); - logCircle(); - } - - private void addNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].put(hash(instance, i).asLong(), instance); - } - } - - private void removeNode(ServerInstance instance) { - for (int i = 0; i < virtualNodesSize; i++) { - circles[instance.getServerAddress().getServerType().ordinal()].remove(hash(instance, i).asLong()); - } - } - - private HashCode hash(ServerInstance instance, int i) { - return hashFunction.newHasher().putString(instance.getHost(), MiscUtils.UTF8).putInt(instance.getPort()).putInt(i).hash(); - } - - private void logCircle() { - log.trace("Consistent Hash Circle Start"); - Arrays.asList(circles).forEach(ConsistentHashCircle::log); - log.trace("Consistent Hash Circle End"); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 0d0e38a1b2..295ce1aa13 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -45,7 +45,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") @Slf4j public class DefaultTbCoreConsumerService implements TbCoreConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 413d5ee9dd..ef2b44bb6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -44,26 +45,26 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") @Slf4j public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { - @Value("${queue.rule-engine.poll_interval}") + @Value("${queue.rule_engine.poll_interval}") private long pollDuration; - @Value("${queue.rule-engine.pack_processing_timeout}") + @Value("${queue.rule_engine.pack_processing_timeout}") private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:false}") + @Value("${queue.rule_engine.stats.enabled:false}") private boolean statsEnabled; private final ActorSystemContext actorContext; - private final TbQueueConsumer> consumer; + private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { - this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + public DefaultTbRuleEngineConsumerService(TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); this.actorContext = actorContext; } @@ -78,17 +79,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); - ConcurrentMap> ackMap = msgs.stream().collect( + List> msgs = consumer.poll(pollDuration); + ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); ackMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { - TransportProtos.ToCoreMsg toCoreMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toCoreMsg); - if (toCoreMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); + if (toRuleEngineMsg.hasToRuleEngineMsg()) { + forwardToRuleEngineActor(toRuleEngineMsg.getToRuleEngineMsg(), callback); } else { callback.onSuccess(); } @@ -112,11 +113,12 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } - private void forwardToDeviceActor(TransportProtos.TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + //TODO 2.5 + private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toDeviceActorMsg, TbMsgCallback callback) { +// if (statsEnabled) { +// stats.log(toDeviceActorMsg); +// } +// actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); } @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 75711d20b2..a5e8ba94af 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,60 +24,29 @@ import java.util.concurrent.atomic.AtomicInteger; public class TbRuleEngineConsumerStats { private final AtomicInteger totalCounter = new AtomicInteger(0); - private final AtomicInteger sessionEventCounter = new AtomicInteger(0); -// private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); -// private final AtomicInteger postAttributesCounter = new AtomicInteger(0); - private final AtomicInteger getAttributesCounter = new AtomicInteger(0); - private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); - private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); - private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); -// private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); - private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); - private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); + private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); + private final AtomicInteger postAttributesCounter = new AtomicInteger(0); + private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); - public void log(TransportProtos.TransportToDeviceActorMsg msg) { + public void log(TransportProtos.TransportToRuleEngineMsg msg) { totalCounter.incrementAndGet(); - if (msg.hasSessionEvent()) { - sessionEventCounter.incrementAndGet(); + if (msg.hasPostTelemetry()) { + postTelemetryCounter.incrementAndGet(); } -// if (msg.hasPostTelemetry()) { -// postTelemetryCounter.incrementAndGet(); -// } -// if (msg.hasPostAttributes()) { -// postAttributesCounter.incrementAndGet(); -// } - if (msg.hasGetAttributes()) { - getAttributesCounter.incrementAndGet(); + if (msg.hasPostAttributes()) { + postAttributesCounter.incrementAndGet(); } - if (msg.hasSubscribeToAttributes()) { - subscribeToAttributesCounter.incrementAndGet(); - } - if (msg.hasSubscribeToRPC()) { - subscribeToRPCCounter.incrementAndGet(); - } - if (msg.hasToDeviceRPCCallResponse()) { - toDeviceRPCCallResponseCounter.incrementAndGet(); - } -// if (msg.hasToServerRPCCallRequest()) { -// toServerRPCCallRequestCounter.incrementAndGet(); -// } - if (msg.hasSubscriptionInfo()) { - subscriptionInfoCounter.incrementAndGet(); - } - if (msg.hasClaimDevice()) { - claimDeviceCounter.incrementAndGet(); + if (msg.hasToServerRPCCallRequest()) { + toServerRPCCallRequestCounter.incrementAndGet(); } } public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + - "toServerRpc [{}] subInfo [{}] claimDevice [{}] ", - total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), - subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + log.info("Transport total [{}] telemetry [{}] attributes [{}] toServerRpc [{}]", + total, postTelemetryCounter.getAndSet(0), + postAttributesCounter.getAndSet(0), toServerRPCCallRequestCounter.getAndSet(0)); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java index 5d303b54ab..8baecbf37a 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -42,8 +42,6 @@ import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -64,12 +62,6 @@ public class DefaultDeviceRpcService implements DeviceRpcService { private static final ObjectMapper json = new ObjectMapper(); - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - @Autowired private DeviceService deviceService; @@ -106,7 +98,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { @Override public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - if (routingService.getCurrentServer().equals(requestOriginAddress)) { + //TODO 2.5 + if (true) {//routingService.getCurrentServer().equals(requestOriginAddress) UUID requestId = response.getId(); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { @@ -124,7 +117,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } else { builder.setError(-1); } - rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); } } @@ -159,7 +153,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); + //TODO 2.5 +// processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); } @Override @@ -172,8 +167,9 @@ public class DefaultDeviceRpcService implements DeviceRpcService { ObjectNode entityNode = json.createObjectNode(); TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("requestUUID", msg.getId().toString()); - metaData.putValue("originHost", routingService.getCurrentServer().getHost()); - metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); + //TODO 2.5 +// metaData.putValue("originHost", routingService.getCurrentServer().getHost()); +// metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); metaData.putValue("oneway", Boolean.toString(msg.isOneway())); @@ -197,9 +193,10 @@ public class DefaultDeviceRpcService implements DeviceRpcService { } private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); - log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); - forward(msg.getDeviceId(), rpcMsg); + //TODO 2.5 +// ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); +// log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); +// forward(msg.getDeviceId(), rpcMsg); } private void forward(DeviceId deviceId, T msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index f40de180ab..31b712644d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -57,21 +57,31 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.ACTIVITY_EVENT; +import static org.thingsboard.server.common.data.DataConstants.CONNECT_EVENT; +import static org.thingsboard.server.common.data.DataConstants.DISCONNECT_EVENT; +import static org.thingsboard.server.common.data.DataConstants.INACTIVITY_EVENT; +import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; /** * Created by ashvayka on 01.05.18. @@ -111,12 +121,6 @@ public class DefaultDeviceStateService implements DeviceStateService { @Autowired private TelemetrySubscriptionService tsSubService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Value("${state.defaultInactivityTimeoutInSec}") @Getter private long defaultInactivityTimeoutInSec; @@ -235,19 +239,20 @@ public class DefaultDeviceStateService implements DeviceStateService { TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(device.getId()).isPresent()) { if (!deviceStates.containsKey(device.getId())) { fetchFutures.add(fetchDeviceState(device)); } - } else { - Set tenantDeviceSet = tenantDevices.get(tenant.getId()); - if (tenantDeviceSet != null) { - tenantDeviceSet.remove(device.getId()); - } - deviceStates.remove(device.getId()); - deviceLastReportedActivity.remove(device.getId()); - deviceLastSavedActivity.remove(device.getId()); - } +// } else { +// Set tenantDeviceSet = tenantDevices.get(tenant.getId()); +// if (tenantDeviceSet != null) { +// tenantDeviceSet.remove(device.getId()); +// } +// deviceStates.remove(device.getId()); +// deviceLastReportedActivity.remove(device.getId()); +// deviceLastSavedActivity.remove(device.getId()); +// } } try { Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); @@ -268,9 +273,10 @@ public class DefaultDeviceStateService implements DeviceStateService { TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - if (!routingService.resolveById(device.getId()).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(device.getId()).isPresent()) { fetchFutures.add(fetchDeviceState(device)); - } +// } } try { Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); @@ -356,7 +362,8 @@ public class DefaultDeviceStateService implements DeviceStateService { private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - if (!routingService.resolveById(deviceId).isPresent()) { + //TODO 2.5 +// if (!routingService.resolveById(deviceId).isPresent()) { Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); if (device != null) { try { @@ -366,7 +373,7 @@ public class DefaultDeviceStateService implements DeviceStateService { log.debug("[{}] Failed to fetch device state!", deviceId, e); } } - } +// } } return deviceStateData; } @@ -389,8 +396,9 @@ public class DefaultDeviceStateService implements DeviceStateService { } private void onDeviceAddedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(device.getId()); +// if (!address.isPresent()) { Futures.addCallback(fetchDeviceState(device), new FutureCallback() { @Override public void onSuccess(@Nullable DeviceStateData state) { @@ -402,9 +410,9 @@ public class DefaultDeviceStateService implements DeviceStateService { log.warn("Failed to register device to the state service", t); } }); - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); - } +// } else { +// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); +// } } private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { @@ -417,12 +425,14 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); } private void onDeviceUpdatedSync(Device device) { - Optional address = routingService.resolveById(device.getId()); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(device.getId()); +// if (!address.isPresent()) { DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); if (stateData != null) { TbMsgMetaData md = new TbMsgMetaData(); @@ -430,14 +440,15 @@ public class DefaultDeviceStateService implements DeviceStateService { md.putValue("deviceType", device.getType()); stateData.setMetaData(md); } - } else { - sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); - } +// } else { +// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); +// } } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - Optional address = routingService.resolveById(deviceId); - if (!address.isPresent()) { + //TODO 2.5 +// Optional address = routingService.resolveById(deviceId); +// if (!address.isPresent()) { deviceStates.remove(deviceId); deviceLastReportedActivity.remove(deviceId); deviceLastSavedActivity.remove(deviceId); @@ -448,9 +459,9 @@ public class DefaultDeviceStateService implements DeviceStateService { tenantDevices.remove(tenantId); } } - } else { - sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); - } +// } else { +// sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); +// } } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d29e5de5b7..04952a3bd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -56,8 +56,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.Subscription; @@ -102,12 +100,6 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio @Autowired private TimeseriesService tsService; - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService rpcService; - @Autowired private EntityViewService entityViewService; @@ -152,7 +144,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio endTime = entityView.getEndTimeMs(); sub = getUpdatedSubscriptionState(entityId, sub, entityView); } - Optional server = routingService.resolveById(entityId); + //TODO 2.5 + Optional server = Optional.empty();//routingService.resolveById(entityId); Subscription subscription; if (server.isPresent()) { ServerAddress address = server.get(); @@ -340,7 +333,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio while (deviceIterator.hasNext()) { Map.Entry> e = deviceIterator.next(); Set subscriptions = e.getValue(); - Optional newAddressOptional = routingService.resolveById(e.getKey()); + //TODO 2.5 + Optional newAddressOptional = Optional.empty();// routingService.resolveById(e.getKey()); if (newAddressOptional.isPresent()) { newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); } else { @@ -424,7 +418,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - Optional serverAddress = routingService.resolveById(entityId); + //TODO 2.5 + Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); if (!serverAddress.isPresent()) { onLocalAttributesUpdate(entityId, scope, attributes); if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { @@ -440,7 +435,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } private void onTimeseriesUpdate(EntityId entityId, List ts) { - Optional serverAddress = routingService.resolveById(entityId); + //TODO 2.5 + Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); if (!serverAddress.isPresent()) { onLocalTimeseriesUpdate(entityId, ts); } else { @@ -632,7 +628,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { @@ -657,7 +654,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.addData(dataBuilder.build()); } ); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { @@ -666,7 +664,8 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.setEntityType(entityId.getEntityType().name()); builder.setScope(scope); attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { @@ -674,17 +673,20 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio builder.setEntityId(entityId.getId().toString()); builder.setEntityType(entityId.getEntityType().name()); ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); } private void tellRemoteSessionClose(ServerAddress address, String sessionId) { ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); } private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); + //TODO 2.5 +// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); } private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 1aee532c93..173c293363 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -24,9 +24,6 @@ import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.cluster.rpc.ClusterRpcService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import javax.annotation.PostConstruct; @@ -50,12 +47,6 @@ import java.util.function.Consumer; @Slf4j public class BaseRuleChainTransactionService implements RuleChainTransactionService { - @Autowired - private ClusterRoutingService routingService; - - @Autowired - private ClusterRpcService clusterRpcService; - @Autowired private DbCallbackExecutorService callbackExecutor; @@ -110,13 +101,14 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ @Override public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); - if (address.isPresent()) { - sendTransactionEventToRemoteServer(msg, address.get()); - executeOnSuccess(onSuccess, msg); - } else { + //TODO 2.5 +// Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); +// if (address.isPresent()) { +// sendTransactionEventToRemoteServer(msg, address.get()); +// executeOnSuccess(onSuccess, msg); +// } else { endLocalTransaction(msg, onSuccess, onFailure); - } +// } } @Override @@ -237,6 +229,7 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); + //TODO 2.5 +// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index fd8ee406c2..f7afd105dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,6 +23,7 @@ import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; @@ -34,17 +35,15 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Slf4j @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core')") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public class DefaultTbCoreToTransportService implements TbCoreToTransportService { - private final TbCoreQueueProvider tbCoreQueueProvider; private final TbQueueProducer> tbTransportProducer; @Value("${queue.notifications.topic}") private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbCoreQueueProvider = tbCoreQueueProvider; this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); } @@ -60,7 +59,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); - tbTransportProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + tbTransportProducer.send(TopicPartitionInfo.builder().topic(topic).build(), queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private static class QueueCallbackAdaptor implements TbQueueCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java deleted file mode 100644 index eed2b8fa8b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import akka.actor.ActorRef; -import io.github.bucket4j.Bandwidth; -import io.github.bucket4j.BlockingBucket; -import io.github.bucket4j.Bucket4j; -import io.github.bucket4j.local.LocalBucket; -import io.github.bucket4j.local.LocalBucketBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.cluster.routing.ClusterRoutingService; -import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.TbCoreConsumerStats; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 09.10.18. - */ -@Slf4j -@Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteRuleEngineTransportService { - - @Value("${transport.remote.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${transport.remote.notifications.topic}") - private String notificationsTopic; - @Value("${transport.remote.rule_engine.poll_interval}") - private int pollDuration; - @Value("${transport.remote.rule_engine.auto_commit_interval}") - private int autoCommitInterval; - - @Value("${transport.remote.rule_engine.poll_records_pack_size}") - private int pollRecordsPackSize; - @Value("${transport.remote.rule_engine.max_poll_records_per_second}") - private long pollRecordsPerSecond; - @Value("${transport.remote.rule_engine.max_poll_records_per_minute}") - private long pollRecordsPerMinute; - @Value("${transport.remote.rule_engine.stats.enabled:false}") - private boolean statsEnabled; - - @Autowired - private ActorSystemContext actorContext; - - //TODO: completely replace this routing with the Kafka routing by partition ids. - @Autowired - private ClusterRoutingService routingService; - @Autowired - private ClusterRpcService rpcService; - @Autowired - private DataDecodingEncodingService encodingService; - - @Autowired - private TbCoreQueueProvider coreQueueProvider; - - private TbQueueConsumer> ruleEngineConsumer; - - private TbQueueProducer> notificationsProducer; - - private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-main-consumer")); - - private volatile boolean stopped = false; - - private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - - @PostConstruct - public void init() { - notificationsProducer = coreQueueProvider.getTransportMsgProducer(); - notificationsProducer.init(); - - //TODO: 2.5 -// ruleEngineConsumer = -// ruleEngineConsumer.subscribe(); - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting polling for events."); - LocalBucketBuilder builder = Bucket4j.builder(); - builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1))); - builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1))); - LocalBucket pollRateBucket = builder.build(); - BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler(); - - mainConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = ruleEngineConsumer.poll(pollDuration); - int recordsCount = msgs.size(); - if (recordsCount > 0) { - while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) { - log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens()); - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - } - log.trace("Processing {} records", recordsCount); - } - msgs.forEach(msg -> { - try { - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToDeviceActorMsg()) { - forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg()); - } - } catch (Throwable e) { - log.warn("Failed to process the notification.", e); - } - }); - } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - @Scheduled(fixedDelayString = "${transport.remote.rule_engine.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - stats.printStats(); - } - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { - process(nodeId, msg, null, null); - } - - @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; - UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - //TODO: 2.5 id - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(sessionId, transportMsg); - notificationsProducer.send(topic, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg); - Optional address = routingService.resolveById(wrapper.getDeviceId()); - if (address.isPresent()) { - log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg); - rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper)); - } else { - log.trace("Pushing message to local server: {}", toDeviceActorMsg); - actorContext.getAppActor().tell(wrapper, ActorRef.noSender()); - } - } - - @PreDestroy - public void destroy() { - stopped = true; - if (ruleEngineConsumer != null) { - ruleEngineConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - } - - private static class QueueCallbackAdaptor implements TbQueueCallback { - private final Runnable onSuccess; - private final Consumer onFailure; - - QueueCallbackAdaptor(Runnable onSuccess, Consumer onFailure) { - this.onSuccess = onSuccess; - this.onFailure = onFailure; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - if (onSuccess != null) { - onSuccess.run(); - } - } - - @Override - public void onFailure(Throwable t) { - if (onFailure != null) { - onFailure.accept(t); - } - } - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index 4765737430..e84e1b595e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -44,7 +44,8 @@ import java.util.concurrent.*; */ @Slf4j @Service -@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") +//TODO 2.5: This Confitional annotation should be removed, and Service renamed to something meaningful +//@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") public class RemoteTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java deleted file mode 100644 index ac1c5965f3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiRequestDecoder.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; - -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiRequestDecoder implements TbKafkaDecoder { - @Override - public TransportApiRequestMsg decode(byte[] data) throws IOException { - return TransportApiRequestMsg.parseFrom(data); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java deleted file mode 100644 index 35f8a7b1aa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiResponseEncoder.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transport; - -import org.thingsboard.server.kafka.TbKafkaEncoder; - -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; - -/** - * Created by ashvayka on 05.10.18. - */ -public class TransportApiResponseEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(TransportApiResponseMsg value) { - return value.toByteArray(); - } -} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index da2e5beb98..57d837de31 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -528,7 +528,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -556,6 +556,7 @@ queue: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java index d717169048..0c34bbf9d6 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,83 +21,92 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.ReflectionTestUtils; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.cluster.ServerType; -import org.thingsboard.server.service.cluster.discovery.DiscoveryService; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; +import org.thingsboard.server.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TbServiceInfoProvider; +import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) public class ConsistentClusterRoutingServiceTest { - private ConsistentClusterRoutingService clusterRoutingService; + public static final int ITERATIONS = 1000000; + private ConsistentHashPartitionService clusterRoutingService; - private DiscoveryService discoveryService; + private TbServiceInfoProvider discoveryService; private String hashFunctionName = "murmur3_128"; - private Integer virtualNodesSize = 1024*4; - private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); + private Integer virtualNodesSize = 16; @Before public void setup() throws Exception { - discoveryService = mock(DiscoveryService.class); - clusterRoutingService = new ConsistentClusterRoutingService(); - ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); + discoveryService = mock(TbServiceInfoProvider.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService); + ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); + ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); + ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); + ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); - when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); - List otherServers = new ArrayList<>(); + TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96.1.1") + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build(); +// when(discoveryService.getServiceInfo()).thenReturn(currentServer); + List otherServers = new ArrayList<>(); for (int i = 1; i < 30; i++) { - otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); + otherServers.add(TransportProtos.ServiceInfo.newBuilder() + .setServiceId("100.96." + i * 2 + "." + i) + .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) + .build()); } - when(discoveryService.getOtherServers()).thenReturn(otherServers); clusterRoutingService.init(); + clusterRoutingService.recalculatePartitions(currentServer, otherServers); } @Test public void testDispersionOnMillionDevices() { List devices = new ArrayList<>(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < ITERATIONS; i++) { devices.add(new DeviceId(UUIDs.timeBased())); } - testDevicesDispersion(devices); } private void testDevicesDispersion(List devices) { long start = System.currentTimeMillis(); - Map map = new HashMap<>(); + Map map = new HashMap<>(); for (DeviceId deviceId : devices) { - ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); - map.put(address, map.getOrDefault(address, 0) + 1); + TopicPartitionInfo address = clusterRoutingService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, deviceId); + Integer partition = address.getPartition().get(); + map.put(partition, map.getOrDefault(partition, 0) + 1); } - List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); + List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); + double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff/ITERATIONS) * 100.0) + "%)"); - for (Map.Entry entry : data) { -// System.out.println(entry.getKey().getHost() + ": " + entry.getValue()); + for (Map.Entry entry : data) { + System.out.println(entry.getKey() + ": " + entry.getValue()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index 26d90a26c6..b3778c2f8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.discovery.TopicPartitionInfo; public interface TbQueueProducer { @@ -23,10 +23,6 @@ public interface TbQueueProducer { String getDefaultTopic(); - void send(T msg, TbQueueCallback callback); - - void send(String topic, T msg, TbQueueCallback callback); - - ListenableFuture send(String topic, int partition, T msg, TbQueueCallback callback); + void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); } diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index d022dafadc..53914953f3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -28,6 +28,7 @@ import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueMsgMetadata; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -160,7 +161,7 @@ public class DefaultTbQueueRequestTemplate responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future); pendingRequests.putIfAbsent(requestId, responseMetaData); log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, request.getKey(), responseMetaData.expTime); - requestTemplate.send(request, new TbQueueCallback() { + requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { log.trace("[{}] Request sent: {}", requestId, metadata); diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java index 3804fb5160..c77692ee36 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java @@ -23,6 +23,7 @@ import org.thingsboard.server.TbQueueHandler; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueResponseTemplate; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -108,7 +109,7 @@ public class DefaultTbQueueResponseTemplate { pendingRequestCount.decrementAndGet(); response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); - responseTemplate.send(responseTopic, response, null); + responseTemplate.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); }, e -> { pendingRequestCount.decrementAndGet(); diff --git a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java index 711677dc78..f96fbd4ece 100644 --- a/application/src/main/java/org/thingsboard/server/service/cluster/routing/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cluster.routing; +package org.thingsboard.server.discovery; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.service.cluster.discovery.ServerInstance; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -25,11 +24,10 @@ import java.util.concurrent.ConcurrentSkipListMap; * Created by ashvayka on 23.09.18. */ @Slf4j -public class ConsistentHashCircle { - private final ConcurrentNavigableMap circle = - new ConcurrentSkipListMap<>(); +public class ConsistentHashCircle { + private final ConcurrentNavigableMap circle = new ConcurrentSkipListMap<>(); - public void put(long hash, ServerInstance instance) { + public void put(long hash, T instance) { circle.put(hash, instance); } @@ -45,7 +43,7 @@ public class ConsistentHashCircle { return circle.containsKey(hash); } - public ConcurrentNavigableMap tailMap(Long hash) { + public ConcurrentNavigableMap tailMap(Long hash) { return circle.tailMap(hash); } @@ -53,11 +51,11 @@ public class ConsistentHashCircle { return circle.firstKey(); } - public ServerInstance get(Long hash) { + public T get(Long hash) { return circle.get(hash); } public void log() { - circle.entrySet().forEach((e) -> log.debug("{} -> {}", e.getKey(), e.getValue().getServerAddress())); + circle.forEach((key, value) -> log.debug("{} -> {}", key, value)); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java new file mode 100644 index 0000000000..c9b7b04e70 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java @@ -0,0 +1,221 @@ +package org.thingsboard.server.discovery; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; + +@Service +@Slf4j +public class ConsistentHashPartitionService implements PartitionService { + + @Value("${queue.core.topic}") + private String coreTopic; + @Value("${queue.core.partitions:100}") + private Integer corePartitions; + @Value("${queue.rule_engine.topic}") + private String ruleEngineTopic; + @Value("${queue.rule_engine.partitions:100}") + private Integer ruleEnginePartitions; + @Value("${queue.partitions.hash_function_name:murmur3_32}") + private String hashFunctionName; + @Value("${queue.partitions.virtual_nodes_size:16}") + private Integer virtualNodesSize; + + private final TbServiceInfoProvider serviceInfoProvider; + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. + private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + + private HashFunction hashFunction; + + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider) { + this.serviceInfoProvider = serviceInfoProvider; + } + + @PostConstruct + public void init() { + this.hashFunction = forName(hashFunctionName); + partitionSizes.put(ServiceType.TB_CORE, corePartitions); + partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); + partitionTopics.put(ServiceType.TB_CORE, coreTopic); + partitionTopics.put(ServiceType.TB_RULE_ENGINE, ruleEngineTopic); + } + + @Override + public List getCurrentPartitions(ServiceType serviceType) { + ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + TenantId tenantId = getTenantId(currentService); + ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); + List partitions = myPartitions.get(serviceKey); + List topicPartitions = new ArrayList<>(); + for (Integer partition : partitions) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (!tenantId.isNullUid()) { + tpi.tenantId(tenantId); + } + topicPartitions.add(tpi.build()); + } + return topicPartitions; + } + + @Override + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + int hash = hashFunction.newHasher() + .putLong(entityId.getId().getMostSignificantBits()) + .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); + int partition = Math.abs(hash % partitionSizes.get(serviceType)); + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (isolated) { + tpi.tenantId(tenantId); + } + return tpi.build(); + } + + @Override + public void recalculatePartitions(ServiceInfo currentService, List otherServices) { + logServiceInfo(currentService); + otherServices.forEach(this::logServiceInfo); + + Map> newCircles = new HashMap<>(ServiceType.values().length); + for (ServiceType serverType : ServiceType.values()) { + newCircles.put(serverType, new ConsistentHashCircle<>()); + } + addNode(newCircles, currentService); + for (ServiceInfo other : otherServices) { + addNode(newCircles, other); + TenantId tenantId = getTenantId(other); + if (!tenantId.isNullUid()) { + isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); + for (String serviceType : other.getServiceTypesList()) { + isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); + } + + } + } + ConcurrentMap> oldPartitions = myPartitions; + myPartitions = new ConcurrentHashMap<>(); + partitionSizes.forEach((type, size) -> { + for (int i = 0; i < size; i++) { + ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); + if (currentService.equals(serviceInfo)) { + myPartitions.putIfAbsent(new ServiceKey(type, getTenantId(serviceInfo)), new ArrayList<>()); + } + } + }); + myPartitions.forEach((serviceKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + } + }); + } + + private void logServiceInfo(TransportProtos.ServiceInfo server) { + TenantId tenantId = getTenantId(server); + if (tenantId.isNullUid()) { + log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); + } else { + log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList()); + } + } + + private TenantId getTenantId(TransportProtos.ServiceInfo serviceInfo) { + return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); + } + + private void addNode(Map> circles, ServiceInfo instance) { + for (String serviceTypeStr : instance.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.get(serviceType).put(hash(instance, i).asLong(), instance); + } + } + } + + private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { + if (circle.isEmpty()) { + return null; + } + Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); + if (!circle.containsKey(hash)) { + ConcurrentNavigableMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? + circle.firstKey() : tailMap.firstKey(); + } + return circle.get(hash); + } + + private HashCode hash(ServiceInfo instance, int i) { + return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); + } + + private static class ServiceKey { + @Getter + private final ServiceType serviceType; + @Getter + private final TenantId tenantId; + + public ServiceKey(ServiceType serviceType, TenantId tenantId) { + this.serviceType = serviceType; + this.tenantId = tenantId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceKey that = (ServiceKey) o; + return serviceType == that.serviceType && + Objects.equals(tenantId, that.tenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, tenantId); + } + } + + public static HashFunction forName(String name) { + switch (name) { + case "murmur3_32": + return Hashing.murmur3_32(); + case "murmur3_128": + return Hashing.murmur3_128(); + case "crc32": + return Hashing.crc32(); + case "md5": + return Hashing.md5(); + default: + throw new IllegalArgumentException("Can't find hash function with name " + name); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java index 0b09a333d8..1fd87a40b6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -69,11 +69,14 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { ServiceInfo.Builder builder = ServiceInfo.newBuilder() .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); + UUID tenantId; if (!StringUtils.isEmpty(tenantIdStr)) { - UUID tenantId = UUID.fromString(tenantIdStr); - builder.setTenantIdMSB(tenantId.getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); + tenantId = UUID.fromString(tenantIdStr); + } else { + tenantId = TenantId.NULL_UUID; } + builder.setTenantIdMSB(tenantId.getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java new file mode 100644 index 0000000000..ef9aca72a0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -0,0 +1,32 @@ +package org.thingsboard.server.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import java.util.Collections; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +@Slf4j +@DependsOn("environmentLogService") +public class DummyDiscoveryService implements PartitionDiscoveryService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + + + public DummyDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java index d85b1d3f44..6fe4571517 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java @@ -15,15 +15,6 @@ */ package org.thingsboard.server.discovery; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.List; - public interface PartitionDiscoveryService { - List getCurrentPartitions(ServiceType serviceType); - - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); - } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java new file mode 100644 index 0000000000..90d43f4af3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java @@ -0,0 +1,16 @@ +package org.thingsboard.server.discovery; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; + +public interface PartitionService { + + List getCurrentPartitions(ServiceType serviceType); + + TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + + void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java index b46b941360..4480d07dd6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java @@ -15,12 +15,27 @@ */ package org.thingsboard.server.discovery; -import lombok.Data; +import lombok.Builder; +import org.thingsboard.server.common.data.id.TenantId; -@Data +import java.util.Optional; + +@Builder public class TopicPartitionInfo { - private String topic; - private int partition; + private final String topic; + private final TenantId tenantId; + private final Integer partition; + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + public Optional getPartition() { + return Optional.ofNullable(partition); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java index a722f2f395..d9af5bb894 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,11 +15,13 @@ */ package org.thingsboard.server.discovery; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.SerializationUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -39,16 +41,21 @@ import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @@ -66,16 +73,8 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P @Value("${zk.zk_dir}") private String zkDir; - @Value("${queue.core.partitions:100}") - private Integer corePartitions; - @Value("${queue.rule_engine.partitions:100}") - private Integer ruleEnginePartitions; - - @Autowired - private TbServiceInfoProvider serviceIdProvider; - - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private final ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private ExecutorService reconnectExecutorService; private CuratorFramework client; @@ -85,14 +84,9 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P private volatile boolean stopped = true; - @Override - public List getCurrentPartitions(ServiceType serviceType) { - return Collections.emptyList(); - } - - @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - + public ZkPartitionDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; + this.partitionService = partitionService; } @PostConstruct @@ -105,15 +99,26 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); - partitionSizes.put(ServiceType.TB_CORE, corePartitions); - partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); - log.info("Initializing discovery service using ZK connect string: {}", zkUrl); zkNodesDir = zkDir + "/nodes"; initZkClient(); } + private List getOtherServers() { + return cache.getCurrentData().stream() + .filter(cd -> !cd.getPath().equals(nodePath)) + .map(cd -> { + try { + return TransportProtos.ServiceInfo.parseFrom(cd.getData()); + } catch (NoSuchElementException | InvalidProtocolBufferException e) { + log.error("Failed to decode ZK node", e); + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { if (stopped) { @@ -127,23 +132,20 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return; } publishCurrentServer(); - getOtherServers().forEach( - server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort()) - ); + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); } - @Override public synchronized void publishCurrentServer() { - ServerInstance self = this.serverInstance.getSelf(); + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); if (currentServerExists()) { - log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath); + log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); } else { try { - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort()); + log.info("[{}] Creating ZK node for current instance", self.getServiceId()); nodePath = client.create() .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress())); - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath); + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", self.toByteArray()); + log.info("[{}] Created ZK node for current instance: {}", self.getServiceId(), nodePath); client.getConnectionStateListenable().addListener(checkReconnect(self)); } catch (Exception e) { log.error("Failed to create ZK node", e); @@ -157,10 +159,10 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return false; } try { - ServerInstance self = this.serverInstance.getSelf(); - ServerAddress registeredServerAdress = null; - registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath)); - if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) { + TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); + TransportProtos.ServiceInfo registeredServerInfo = null; + registeredServerInfo = TransportProtos.ServiceInfo.parseFrom(client.getData().forPath(nodePath)); + if (self.equals(registeredServerInfo)) { return true; } } catch (KeeperException.NoNodeException e) { @@ -171,9 +173,9 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P return false; } - private ConnectionStateListener checkReconnect(ServerInstance self) { + private ConnectionStateListener checkReconnect(TransportProtos.ServiceInfo self) { return (client, newState) -> { - log.info("[{}:{}] ZK state changed: {}", self.getHost(), self.getPort(), newState); + log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); if (newState == ConnectionState.LOST) { reconnectExecutorService.submit(this::reconnect); } @@ -250,6 +252,46 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P @Override public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception { - + if (stopped) { + log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent); + return; + } + if (client.getState() != CuratorFrameworkState.STARTED) { + log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState()); + return; + } + ChildData data = pathChildrenCacheEvent.getData(); + if (data == null) { + log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent); + return; + } else if (data.getData() == null) { + log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent); + return; + } else if (nodePath != null && nodePath.equals(data.getPath())) { + if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) { + log.info("ZK node for current instance is somehow deleted."); + publishCurrentServer(); + } + log.debug("Ignoring event about current server {}", pathChildrenCacheEvent); + return; + } + TransportProtos.ServiceInfo instance; + try { + instance = TransportProtos.ServiceInfo.parseFrom(data.getData()); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode server instance for node {}", data.getPath(), e); + throw e; + } + log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); + switch (pathChildrenCacheEvent.getType()) { + case CHILD_ADDED: + case CHILD_UPDATED: + case CHILD_REMOVED: + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + break; + default: + break; + } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java index a6a881ecde..cff3fd1056 100644 --- a/application/src/main/java/org/thingsboard/server/service/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.environment; +package org.thingsboard.server.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; @@ -33,7 +33,7 @@ public class EnvironmentLogService { @PostConstruct public void init() { - Environment.logEnv("Thingsboard server environment: ", log); + Environment.logEnv("ThingsBoard server environment: ", log); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index d76a2dc625..8072a2da66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.discovery.TopicPartitionInfo; import java.util.List; import java.util.Properties; @@ -46,10 +47,6 @@ public class TBKafkaProducerTemplate implements TbQueuePro private final KafkaProducer producer; - private final TbKafkaPartitioner partitioner; - - private ConcurrentMap> partitionInfoMap; - @Getter private final String defaultTopic; @@ -66,41 +63,28 @@ public class TBKafkaProducerTemplate implements TbQueuePro } this.settings = settings; this.producer = new KafkaProducer<>(props); - this.partitioner = partitioner; this.defaultTopic = defaultTopic; } - public void init() { - this.partitionInfoMap = new ConcurrentHashMap<>(); - if (!StringUtils.isEmpty(defaultTopic)) { - try { - TBKafkaAdmin admin = new TBKafkaAdmin(this.settings); - admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS); - log.info("[{}] Topic exists.", defaultTopic); - } catch (Exception e) { - log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e); - throw new RuntimeException(e); - } - //Maybe this should not be cached, but we don't plan to change size of partitions - this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic)); - } - } - @Override - public void send(T msg, TbQueueCallback callback) { - send(defaultTopic, msg, callback); + public void init() { } @Override - public void send(String topic, T msg, TbQueueCallback callback) { + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - - Integer partition = getPartition(topic, msg); - record = new ProducerRecord<>(topic, partition, key, data, headers); - Future result = producer.send(record, (metadata, exception) -> { + StringBuilder topic = new StringBuilder().append(tpi.getTopic()); + if (tpi.getTenantId().isPresent()) { + topic.append(".").append(tpi.getTenantId().get().getId().toString()); + } + if (tpi.getPartition().isPresent()) { + topic.append(".").append(tpi.getPartition().get()); + } + record = new ProducerRecord<>(topic.toString(), null, key, data, headers); + producer.send(record, (metadata, exception) -> { if (exception == null) { callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); } else { @@ -108,12 +92,4 @@ public class TBKafkaProducerTemplate implements TbQueuePro } }); } - - private Integer getPartition(String topic, T value) { - if (partitioner == null) { - return null; - } else { - return partitioner.partition(topic, value.getKey().toString(), value, value.getData(), partitionInfoMap.computeIfAbsent(topic, producer::partitionsFor)); - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index e71d579c7f..8abb5e2d43 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ import lombok.Data; import org.thingsboard.server.TbQueueCallback; import org.thingsboard.server.TbQueueMsg; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.discovery.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { @@ -37,18 +38,8 @@ public class InMemoryTbQueueProducer implements TbQueuePro } @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(T msg, TbQueueCallback callback) { - send(defaultTopic, msg, callback); - } - - @Override - public void send(String topic, T msg, TbQueueCallback callback) { - boolean result = storage.put(topic, msg); + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + boolean result = storage.put(tpi.getTopic(), msg); if (result) { callback.onSuccess(null); } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index 79f5dd7da0..c7839e70e9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -32,12 +32,12 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; @Slf4j @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='in-memory'") -public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbQueueCoreSettings coreSettings; - public InMemoryTbCoreQueueProvider(TbQueueCoreSettings coreSettings) { + public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings) { this.coreSettings = coreSettings; } @@ -59,6 +59,12 @@ public class InMemoryTbCoreQueueProvider implements TbCoreQueueProvider { return producer; } + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + return consumer; + } + @Override public TbQueueConsumer> getToCoreMsgConsumer() { InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java new file mode 100644 index 0000000000..6aaf76420b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + //TODO 2.5 + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index d830cff8fa..426a30eea1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +32,7 @@ import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${queue.type:null}'=='kafka'") +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbKafkaSettings kafkaSettings; @@ -49,7 +49,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getTransportMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,7 +58,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -74,28 +74,33 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { @Override public TbQueueConsumer> getToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(coreSettings.getTopic()); + consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(coreSettings.getTopic()); responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO: 2.5 -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); + //TODO 2.5 + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } - @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { - return null; - } - @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..6057ebba6f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.kafka.TbKafkaSettings; +import org.thingsboard.server.kafka.TbNodeIdProvider; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final TbKafkaSettings kafkaSettings; + private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + + public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + this.kafkaSettings = kafkaSettings; + this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + } + + @Override + public TbQueueProducer> getTransportMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(coreSettings.getTopic()); + responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); + responseBuilder.autoCommit(true); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index 3b671a3adb..c7b5b6f839 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -35,7 +35,7 @@ import org.thingsboard.server.kafka.TbKafkaSettings; import org.thingsboard.server.kafka.TbNodeIdProvider; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='kafka'") +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class KafkaTransportQueueProvider implements TransportQueueProvider { @@ -62,9 +62,7 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO: 2.5 -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new TransportApiResponseDecoder()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..46f1a3351d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.provider; + +import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Core Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbRuleEngineQueueProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to consume messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToRuleEngineMsgConsumer(); + +} diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index d6dcbbabc6..af471e537c 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -36,6 +36,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common data diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index cdf6b3f763..661411cf89 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,6 +34,9 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.discovery.PartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -76,8 +79,8 @@ public class DefaultTransportService implements TransportService { @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; - @Autowired - private TransportQueueProvider queueProvider; + private final TransportQueueProvider queueProvider; + private final PartitionService partitionService; @Value("${kafka.notifications.poll_interval}") private int notificationsPollDuration; @@ -98,6 +101,11 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; + public DefaultTransportService(TransportQueueProvider queueProvider, PartitionService partitionService) { + this.queueProvider = queueProvider; + this.partitionService = partitionService; + } + @PostConstruct public void init() { if (rateLimitEnabled) { @@ -420,6 +428,14 @@ public class DefaultTransportService implements TransportService { return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()); } + protected TenantId getTenantId(TransportProtos.SessionInfoProto sessionInfo) { + return new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + } + + protected DeviceId getDeviceId(TransportProtos.SessionInfoProto sessionInfo) { + return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + } + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) { return TransportProtos.SessionEventMsg.newBuilder() .setSessionType(TransportProtos.SessionType.ASYNC) @@ -427,15 +443,19 @@ public class DefaultTransportService implements TransportService { } protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { - tbCoreMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + tbCoreMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); } protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - ruleEngineMsgProducer.send(new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + ruleEngineMsgProducer.send(tpi, + new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), + ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? + new TransportTbQueueCallback(callback) : null); } private class TransportTbQueueCallback implements TbQueueCallback { From 2a815058e7279ee4aefbcaa2c5ba3adff00b7532 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 12:38:28 +0200 Subject: [PATCH 013/196] Improvements and bugfix to QueueProviders --- .../server/actors/ActorSystemContext.java | 23 ++++---- .../actors/ruleChain/DefaultTbContext.java | 4 +- .../RuleChainActorMessageProcessor.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 11 +++- .../DefaultTbRuleEngineConsumerService.java | 16 ++++-- .../service/queue/TbCoreConsumerService.java | 4 +- .../queue/TbRuleEngineConsumerService.java | 5 +- .../queue/TbRuleEngineConsumerStats.java | 2 +- .../DefaultTbCoreToTransportService.java | 4 +- .../src/main/resources/thingsboard.yml | 21 +++----- ...=> ConsistentHashParitionServiceTest.java} | 13 ++--- .../thingsboard/server/TbQueueConsumer.java | 2 + .../thingsboard/server/TbQueueProducer.java | 2 +- .../server/TbQueueRequestTemplate.java | 4 ++ .../server/TbQueueRuleEngineSettings.java | 31 +++++++++++ .../server/TbQueueTransportApiSettings.java | 7 +++ .../TbQueueTransportNotificationSettings.java | 32 +++++++++++ .../common/DefaultTbQueueRequestTemplate.java | 4 +- .../discovery/ConsistentHashCircle.java | 2 +- .../ConsistentHashPartitionService.java | 54 ++++++++----------- .../DefaultTbServiceInfoProvider.java | 2 +- .../discovery/DummyDiscoveryService.java | 15 ++++++ .../discovery/PartitionChangeEvent.java | 6 ++- .../server/discovery/PartitionService.java | 15 ++++++ .../server/discovery/ServiceKey.java | 47 ++++++++++++++++ .../ZkPartitionDiscoveryService.java | 2 +- .../server/kafka/TBKafkaConsumerTemplate.java | 40 +++++++++----- .../server/kafka/TBKafkaProducerTemplate.java | 2 +- .../server/memory/InMemoryStorage.java | 39 +++++++++++--- .../memory/InMemoryTbQueueConsumer.java | 8 ++- .../memory/InMemoryTbQueueProducer.java | 11 ++-- .../InMemoryMonolithQueueProvider.java | 36 +++++++------ .../InMemoryTransportQueueProvider.java | 28 +++++++--- .../provider/KafkaMonolithQueueProvider.java | 4 +- .../provider/KafkaTbCoreQueueProvider.java | 4 +- .../KafkaTbRuleEngineQueueProvider.java | 2 +- .../provider/KafkaTransportQueueProvider.java | 2 +- .../server/provider/TbCoreQueueProvider.java | 2 + .../provider/TbRuleEngineQueueProvider.java | 2 +- .../service/DefaultTransportService.java | 12 +++-- 40 files changed, 384 insertions(+), 138 deletions(-) rename application/src/test/java/org/thingsboard/server/service/cluster/routing/{ConsistentClusterRoutingServiceTest.java => ConsistentHashParitionServiceTest.java} (92%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e5a0e6a25a..3db5c82ee2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -35,6 +35,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; @@ -64,6 +65,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.discovery.TbServiceInfoProvider; import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; @@ -102,6 +104,11 @@ public class ActorSystemContext { return debugPerTenantLimits; } + @Autowired + @Getter + @Setter + private TbServiceInfoProvider serviceInfoProvider; + @Getter @Setter private ActorService actorService; @@ -233,10 +240,6 @@ public class ActorSystemContext { @Getter private RuleChainTransactionService ruleChainTransactionService; - @Value("${cluster.partition_id}") - @Getter - private long queuePartitionId; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -359,7 +362,7 @@ public class ActorSystemContext { event.setEntityId(entityId); event.setType(DataConstants.ERROR); //TODO 2.5 -// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), method, toString(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -369,7 +372,7 @@ public class ActorSystemContext { event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); //TODO 2.5 -// event.setBody(toBodyJson(discoveryService.getCurrentServer().getServerAddress(), lcEvent, Optional.ofNullable(e))); + event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -383,8 +386,8 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(ServerAddress server, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", server.toString()).put("event", event.name()); + private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { + ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); if (e.isPresent()) { node = node.put("success", false); node = node.put("error", toString(e.get())); @@ -394,8 +397,8 @@ public class ActorSystemContext { return node; } - private JsonNode toBodyJson(ServerAddress server, String method, String body) { - return mapper.createObjectNode().put("server", server.toString()).put("method", method).put("error", body); + private JsonNode toBodyJson(String serviceId, String method, String body) { + return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index f5ea5c399f..3e982b520e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -146,12 +146,12 @@ class DefaultTbContext implements TbContext { @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), mainCtx.getQueuePartitionId()); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), 0); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), mainCtx.getQueuePartitionId()); + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), 0); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 095af7f62d..fe95c448a0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -331,6 +331,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ef2b44bb6c..6f8d9dc9ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import akka.actor.ActorRef; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -27,10 +26,10 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.provider.TbCoreQueueProvider; import org.thingsboard.server.provider.TbRuleEngineQueueProvider; -import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -70,10 +69,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @PostConstruct public void init() { - this.consumer.subscribe(); this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); } + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_RULE_ENGINE) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.consumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { mainConsumerExecutor.execute(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 1e1b352645..8a39b5df4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.queue; +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.function.Consumer; -public interface TbCoreConsumerService { +public interface TbCoreConsumerService extends ApplicationListener { } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 671cd72262..968fbb2274 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.service.queue; -public interface TbRuleEngineConsumerService { +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.discovery.PartitionChangeEvent; + +public interface TbRuleEngineConsumerService extends ApplicationListener { } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index a5e8ba94af..27792c5a61 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index f7afd105dc..a6e1a37eca 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -40,7 +40,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService private final TbQueueProducer> tbTransportProducer; - @Value("${queue.notifications.topic}") + @Value("${queue.transport.notifications_topic}") private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 57d837de31..fc49716d25 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -70,16 +70,7 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -# Clustering properties related to consistent-hashing. See architecture docs for more details. cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" - # Name of hash function used for consistent hash ring. - hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}" - # Amount of virtual nodes in consistent hash ring. - vitrual_nodes_size: "${CLUSTER_VIRTUAL_NODES_SIZE:16}" - # Queue partition id for current node - partition_id: "${QUEUE_PARTITION_ID:0}" stats: enabled: "${TB_CLUSTER_STATS_ENABLED:false}" print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" @@ -536,6 +527,9 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" @@ -547,7 +541,7 @@ queue: core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" @@ -555,13 +549,14 @@ queue: rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - notifications: - topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + transport: + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java similarity index 92% rename from application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 0c34bbf9d6..9442202688 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentClusterRoutingServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,11 +21,10 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; import org.thingsboard.server.discovery.ConsistentHashPartitionService; import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.discovery.TbServiceInfoProvider; @@ -44,12 +43,13 @@ import static org.mockito.Mockito.mock; @Slf4j @RunWith(MockitoJUnitRunner.class) -public class ConsistentClusterRoutingServiceTest { +public class ConsistentHashParitionServiceTest { public static final int ITERATIONS = 1000000; private ConsistentHashPartitionService clusterRoutingService; private TbServiceInfoProvider discoveryService; + private ApplicationEventPublisher applicationEventPublisher; private String hashFunctionName = "murmur3_128"; private Integer virtualNodesSize = 16; @@ -58,7 +58,8 @@ public class ConsistentClusterRoutingServiceTest { @Before public void setup() throws Exception { discoveryService = mock(TbServiceInfoProvider.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService); + applicationEventPublisher = mock(ApplicationEventPublisher.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, applicationEventPublisher); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); @@ -103,7 +104,7 @@ public class ConsistentClusterRoutingServiceTest { List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff/ITERATIONS) * 100.0) + "%)"); + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); for (Map.Entry entry : data) { System.out.println(entry.getKey() + ": " + entry.getValue()); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java index ddf9d7d9b3..ac4d7611f6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java @@ -23,6 +23,8 @@ public interface TbQueueConsumer { void subscribe(); + void subscribe(List partitions); + void unsubscribe(); List poll(long durationInMillis); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java index b3778c2f8e..2498117c46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java index 5182bb7260..2eb429b082 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java @@ -19,6 +19,10 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TbQueueRequestTemplate { + void init(); + ListenableFuture send(Request request); + void stop(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java new file mode 100644 index 0000000000..4acfec1091 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueRuleEngineSettings { + + @Value("${queue.rule_engine.topic}") + private String topic; + + @Value("${queue.rule_engine.partitions}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java index 603d787fa8..b98bd3b76d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; @Data @Component public class TbQueueTransportApiSettings { + @Value("${queue.transport_api.requests_topic}") private String requestsTopic; @@ -34,6 +35,12 @@ public class TbQueueTransportApiSettings { @Value("${queue.transport_api.max_requests_timeout}") private int maxRequestsTimeout; + @Value("${queue.transport_api.max_callback_threads}") + private int maxCallbackThreads; + + @Value("${queue.transport_api.request_poll_interval}") + private long requestPollInterval; + @Value("${queue.transport_api.response_poll_interval}") private long responsePollInterval; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java new file mode 100644 index 0000000000..2666045e7e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueTransportNotificationSettings { + + @Value("${queue.transport.notifications_topic}") + private String notificationsTopic; + + @Value("${queue.transport.poll_interval}") + private long transportPollInterval; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java index 53914953f3..0388daebb6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -79,6 +79,7 @@ public class DefaultTbQueueRequestTemplate partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); @@ -53,8 +68,9 @@ public class ConsistentHashPartitionService implements PartitionService { private HashFunction hashFunction; - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider) { + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, ApplicationEventPublisher applicationEventPublisher) { this.serviceInfoProvider = serviceInfoProvider; + this.applicationEventPublisher = applicationEventPublisher; } @PostConstruct @@ -128,13 +144,15 @@ public class ConsistentHashPartitionService implements PartitionService { for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); if (currentService.equals(serviceInfo)) { - myPartitions.putIfAbsent(new ServiceKey(type, getTenantId(serviceInfo)), new ArrayList<>()); + ServiceKey serviceKey = new ServiceKey(type, getTenantId(serviceInfo)); + myPartitions.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(i); } } }); myPartitions.forEach((serviceKey, partitions) -> { if (!partitions.equals(oldPartitions.get(serviceKey))) { log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, partitions)); } }); } @@ -178,32 +196,6 @@ public class ConsistentHashPartitionService implements PartitionService { return hashFunction.newHasher().putString(instance.getServiceId(), StandardCharsets.UTF_8).putInt(i).hash(); } - private static class ServiceKey { - @Getter - private final ServiceType serviceType; - @Getter - private final TenantId tenantId; - - public ServiceKey(ServiceType serviceType, TenantId tenantId) { - this.serviceType = serviceType; - this.tenantId = tenantId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceKey that = (ServiceKey) o; - return serviceType == that.serviceType && - Objects.equals(tenantId, that.tenantId); - } - - @Override - public int hashCode() { - return Objects.hash(serviceType, tenantId); - } - } - public static HashFunction forName(String name) { switch (name) { case "murmur3_32": diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java index 1fd87a40b6..80040a017b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java index ef9aca72a0..8983cf1e97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.discovery; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java index 7d083ba873..570a774c9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java @@ -16,6 +16,7 @@ package org.thingsboard.server.discovery; import lombok.Getter; +import lombok.Setter; import org.springframework.context.ApplicationEvent; import java.util.List; @@ -23,11 +24,14 @@ import java.util.List; public class PartitionChangeEvent extends ApplicationEvent { + @Getter + private final ServiceKey serviceKey; @Getter private final List partitions; - public PartitionChangeEvent(Object source, List partitions) { + public PartitionChangeEvent(Object source, ServiceKey serviceKey, List partitions) { super(source); + this.serviceKey = serviceKey; this.partitions = partitions; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java index 90d43f4af3..ad61a63807 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.discovery; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java new file mode 100644 index 0000000000..0179be968c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.discovery; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +public class ServiceKey { + @Getter + private final ServiceType serviceType; + @Getter + private final TenantId tenantId; + + public ServiceKey(ServiceType serviceType, TenantId tenantId) { + this.serviceType = serviceType; + this.tenantId = tenantId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceKey that = (ServiceKey) o; + return serviceType == that.serviceType && + Objects.equals(tenantId, that.tenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, tenantId); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java index d9af5bb894..07f11617ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 558d556d09..51b8725118 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -40,6 +41,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; + private volatile boolean subscribed; @Getter private final String topic; @@ -69,23 +71,37 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { consumer.subscribe(Collections.singletonList(topic)); + subscribed = true; + } + + @Override + public void subscribe(List partitions) { + consumer.subscribe(partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList())); + subscribed = true; } @Override public List poll(long durationInMillis) { - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; + if (!subscribed) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } } - return Collections.emptyList(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 8072a2da66..1b5d57f1d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java index ded4cd9810..86d103ea70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java @@ -15,16 +15,24 @@ */ package org.thingsboard.server.memory; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.TbQueueMsg; +import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +@Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; - private final Map> storage; + private final Map> storage; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); @@ -42,19 +50,38 @@ public final class InMemoryStorage { } public boolean put(String topic, TbQueueMsg msg) { - return storage.computeIfAbsent(topic, (t) -> new LinkedList<>()).add(msg); + return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public TbQueueMsg get(String topic) { + public List get(String topic, long durationInMillis) { if (storage.containsKey(topic)) { - return storage.get(topic).peek(); + try { + List entities; + T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + } else { + entities = Collections.emptyList(); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 100); + for (TbQueueMsg other : otherList) { + entities.add((T) other); + } + } + return entities; + } catch (InterruptedException e) { + log.warn("Queue was interrupted", e); + return Collections.emptyList(); + } } - return null; + return Collections.emptyList(); } public void commit(String topic) { + //TODO: 2.5 Until someone calls commit you should not allow to poll new elements. if (storage.containsKey(topic)) { - storage.get(topic).remove(); +// storage.get(topic).remove(); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java index b8ddcff89c..3f1e844efc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java @@ -18,7 +18,6 @@ package org.thingsboard.server.memory; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueMsg; -import java.util.Collections; import java.util.List; public class InMemoryTbQueueConsumer implements TbQueueConsumer { @@ -40,6 +39,11 @@ public class InMemoryTbQueueConsumer implements TbQueueCon } + @Override + public void subscribe(List partitions) { + + } + @Override public void unsubscribe() { @@ -47,7 +51,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public List poll(long durationInMillis) { - return Collections.singletonList((T)storage.get(topic)); + return storage.get(topic, durationInMillis); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java index 8abb5e2d43..79fa1d6b39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -41,10 +41,13 @@ public class InMemoryTbQueueProducer implements TbQueuePro public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { boolean result = storage.put(tpi.getTopic(), msg); if (result) { - callback.onSuccess(null); + if (callback != null) { + callback.onSuccess(null); + } } else { - Exception e = new RuntimeException("Failure add msg to InMemoryQueue"); - callback.onFailure(e); + if (callback != null) { + callback.onFailure(new RuntimeException("Failure add msg to InMemoryQueue")); + } } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index c7839e70e9..4fa10b7c44 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -21,7 +21,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRuleEngineSettings; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -36,50 +40,52 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings) { + public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.notificationSettings = notificationSettings; } @Override public TbQueueProducer> getTransportMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> getRuleEngineMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); } @Override public TbQueueConsumer> getToCoreMsgConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); } @Override public TbQueueConsumer> getTransportApiRequestConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> getTransportApiResponseProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(coreSettings.getTopic()); - return producer; + return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java index 789ea1cf5a..2d7205e516 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java @@ -15,13 +15,19 @@ */ package org.thingsboard.server.provider; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.TbQueueAdmin; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueRuleEngineSettings; import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -37,20 +43,29 @@ import org.thingsboard.server.memory.InMemoryTbQueueProducer; @Slf4j public class InMemoryTransportQueueProvider implements TransportQueueProvider { + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings) { + public InMemoryTransportQueueProvider(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; + this.notificationSettings = notificationSettings; } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(topic -> Futures.immediateFuture(null)); templateBuilder.requestTemplate(producer); templateBuilder.responseTemplate(consumer); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -61,19 +76,16 @@ public class InMemoryTransportQueueProvider implements TransportQueueProvider { @Override public TbQueueProducer> getRuleEngineMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - return producer; + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - return producer; + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override public TbQueueConsumer> getTransportNotificationsConsumer() { - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); - return consumer; + return new InMemoryTbQueueConsumer<>(notificationSettings.getNotificationsTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java index 6aaf76420b..f06897ebe6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,8 @@ import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 426a30eea1..5cbe93196a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,8 @@ import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java index 6057ebba6f..eb0bcb5d6e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index c7b5b6f839..dcf6474e1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index 6123fc0859..ef9cd23ac3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.provider; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java index 46f1a3351d..2efa9ffbc6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 661411cf89..19bf43f7db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -78,13 +78,12 @@ public class DefaultTransportService implements TransportService { private long sessionInactivityTimeout; @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; + @Value("${queue.transport.poll_interval}") + private int notificationsPollDuration; private final TransportQueueProvider queueProvider; private final PartitionService partitionService; - @Value("${kafka.notifications.poll_interval}") - private int notificationsPollDuration; - protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; protected TbQueueProducer> tbCoreMsgProducer; @@ -120,7 +119,7 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); - + transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { while (!stopped) { try { @@ -163,6 +162,9 @@ public class DefaultTransportService implements TransportService { if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } + if (transportApiRequestTemplate != null) { + transportApiRequestTemplate.stop(); + } } @Override From ea4f35637660d82cc925a4f1cb46541477cb205f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 12:51:03 +0200 Subject: [PATCH 014/196] Consumer improvements --- .../server/service/queue/DefaultTbCoreConsumerService.java | 3 +++ .../service/queue/DefaultTbRuleEngineConsumerService.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 0907b8be60..1d3cf11f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -88,6 +88,9 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { while (!stopped) { try { List> msgs = consumer.poll(pollDuration); + if(msgs.isEmpty()){ + continue; + } ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 6f8d9dc9ac..253c7d9933 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -86,6 +86,9 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS while (!stopped) { try { List> msgs = consumer.poll(pollDuration); + if(msgs.isEmpty()){ + continue; + } ConcurrentMap> ackMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); @@ -120,7 +123,8 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } //TODO 2.5 - private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toDeviceActorMsg, TbMsgCallback callback) { + private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + log.info("Received RULE ENGINE msg: {}", toRuleEngineMsg); // if (statsEnabled) { // stats.log(toDeviceActorMsg); // } From 7de485f4535c86e6cbb3ddafb44d1784898f8d4f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 14:26:44 +0200 Subject: [PATCH 015/196] Kafka configuration --- .../DefaultTbCoreToTransportService.java | 2 +- .../org/thingsboard/server/TbQueueAdmin.java | 4 +- .../server/kafka/TBKafkaAdmin.java | 37 ++++++++--- .../server/kafka/TBKafkaConsumerTemplate.java | 11 +++- .../server/kafka/TBKafkaProducerTemplate.java | 10 ++- .../server/kafka/TbKafkaSettings.java | 5 +- .../InMemoryMonolithQueueProvider.java | 2 +- .../provider/KafkaMonolithQueueProvider.java | 63 +++++++++++-------- .../provider/KafkaTbCoreQueueProvider.java | 5 +- .../KafkaTbRuleEngineQueueProvider.java | 2 +- .../provider/KafkaTransportQueueProvider.java | 39 +++++++++--- .../server/provider/TbCoreQueueProvider.java | 2 +- .../provider/TbRuleEngineQueueProvider.java | 2 +- .../service/DefaultTransportService.java | 1 + 14 files changed, 122 insertions(+), 63 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index a6e1a37eca..833b28c3b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -44,7 +44,7 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService private String notificationsTopic; public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbTransportProducer = tbCoreQueueProvider.getTransportMsgProducer(); + this.tbTransportProducer = tbCoreQueueProvider.getTransportNotificationsMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java index 6917c23719..5952e7da70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java @@ -15,10 +15,8 @@ */ package org.thingsboard.server; -import com.google.common.util.concurrent.ListenableFuture; - public interface TbQueueAdmin { - ListenableFuture createTopicIfNotExists(String topic); + void createTopicIfNotExists(String topic); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java index 0bb3a75db6..5f1b41ea7b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java @@ -18,11 +18,13 @@ package org.thingsboard.server.kafka; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.JdkFutureAdapters; import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.TbQueueAdmin; import java.util.Collections; @@ -33,6 +35,7 @@ import java.util.concurrent.TimeoutException; /** * Created by ashvayka on 24.09.18. */ +@Slf4j public class TBKafkaAdmin implements TbQueueAdmin { AdminClient client; @@ -41,17 +44,31 @@ public class TBKafkaAdmin implements TbQueueAdmin { client = AdminClient.create(settings.toProps()); } + //TODO 2.5 @Override - public ListenableFuture createTopicIfNotExists(String topic) { - - KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); - - ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); - - return Futures.transformAsync(topicFuture, topicDescription -> { - KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); - return JdkFutureAdapters.listenInPoolThread(resultFuture); - }); + public void createTopicIfNotExists(String topic) { + try { + createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic).get(); + } catch (ExecutionException ee) { + if (ee.getCause() instanceof TopicExistsException) { + //do nothing + } else { + log.warn("[{}] Failed to create topic", topic, ee); + throw new RuntimeException(ee); + } + } catch (Exception e) { + log.warn("[{}] Failed to create topic", topic, e); + throw new RuntimeException(e); + } +// +// KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); +// +// ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); +// +// return Futures.transformAsync(topicFuture, topicDescription -> { +// KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); +// return JdkFutureAdapters.listenInPoolThread(resultFuture); +// }); } public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java index 51b8725118..eb35c583c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; /** @@ -39,6 +40,7 @@ import java.util.stream.Collectors; @Slf4j public class TBKafkaConsumerTemplate implements TbQueueConsumer { + private final TBKafkaAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private volatile boolean subscribed; @@ -63,6 +65,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon if (maxPollRecords > 0) { props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); } + this.admin = new TBKafkaAdmin(settings); this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.topic = topic; @@ -70,13 +73,16 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { + createTopicIfNotExists(topic); consumer.subscribe(Collections.singletonList(topic)); subscribed = true; } @Override public void subscribe(List partitions) { - consumer.subscribe(partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList())); + List topicNames = partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList()); + topicNames.forEach(this::createTopicIfNotExists); + consumer.subscribe(topicNames); subscribed = true; } @@ -119,4 +125,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon return decoder.decode(new KafkaTbQueueMsg(record)); } + private void createTopicIfNotExists(String topic) { + admin.createTopicIfNotExists(topic); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java index 1b5d57f1d2..4999ba026a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java @@ -86,9 +86,15 @@ public class TBKafkaProducerTemplate implements TbQueuePro record = new ProducerRecord<>(topic.toString(), null, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { - callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + if (callback != null) { + callback.onSuccess(new KafkaTbQueueMsgMetadata(metadata)); + } } else { - callback.onFailure(exception); + if (callback != null) { + callback.onFailure(exception); + } else { + log.warn("Producer template failure", exception); + } } }); } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java index 566766fee6..912e935bef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java @@ -18,6 +18,7 @@ package org.thingsboard.server.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -28,7 +29,7 @@ import java.util.Properties; * Created by ashvayka on 25.09.18. */ @Slf4j -@ConditionalOnProperty(prefix = "kafka", value = "enabled", havingValue = "true", matchIfMissing = false) +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") @Component public class TbKafkaSettings { @@ -41,7 +42,7 @@ public class TbKafkaSettings { @Value("${queue.kafka.acks}") private String acks; - @Value("${queue.queue.kafka.retries}") + @Value("${queue.kafka.retries}") private int retries; @Value("${queue.kafka.batch.size}") diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java index 4fa10b7c44..7fc7bdea7c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java @@ -55,7 +55,7 @@ public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRul } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java index f06897ebe6..01e7a211d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -20,9 +20,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; +import org.thingsboard.server.TbQueueRuleEngineSettings; +import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -40,19 +41,30 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbKafkaSettings kafkaSettings; private final TbNodeIdProvider nodeIdProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, + TbNodeIdProvider nodeIdProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.nodeIdProvider = nodeIdProvider; this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.clientId("producer-transport-notifications" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); return requestBuilder.build(); } @@ -61,7 +73,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); return requestBuilder.build(); } @@ -76,14 +88,13 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } @Override @@ -92,30 +103,28 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("tb-core-consumer-" + nodeIdProvider.getNodeId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override public TbQueueConsumer> getTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - //TODO 2.5 - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); + consumerBuilder.groupId("transport-api-consumer-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.clientId("transport-api-producer-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); return requestBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java index 5cbe93196a..72338bb149 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java @@ -21,8 +21,6 @@ import org.thingsboard.server.TbQueueConsumer; import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -48,7 +46,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); @@ -93,7 +91,6 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); responseBuilder.autoCommit(true); - //TODO 2.5 responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java index eb0bcb5d6e..d635419141 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java @@ -46,7 +46,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getTransportMsgProducer() { + public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java index dcf6474e1e..577c9180fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java @@ -19,9 +19,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.TbQueueCoreSettings; import org.thingsboard.server.TbQueueProducer; import org.thingsboard.server.TbQueueRequestTemplate; +import org.thingsboard.server.TbQueueRuleEngineSettings; import org.thingsboard.server.TbQueueTransportApiSettings; +import org.thingsboard.server.TbQueueTransportNotificationSettings; import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -29,6 +32,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.kafka.TBKafkaAdmin; import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.kafka.TbKafkaSettings; @@ -41,31 +45,42 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { private final TbKafkaSettings kafkaSettings; private final TbNodeIdProvider nodeIdProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueTransportApiSettings transportApiSettings) { + public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, + TbNodeIdProvider nodeIdProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.nodeIdProvider = nodeIdProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-transport-api-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiSettings.getResponsesTopic()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); + responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("transport-node-" + nodeIdProvider.getNodeId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); templateBuilder.requestTemplate(requestBuilder.build()); templateBuilder.responseTemplate(responseBuilder.build()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -78,7 +93,7 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); return requestBuilder.build(); } @@ -87,13 +102,19 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("producer-tb-core-" + nodeIdProvider.getNodeId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); return requestBuilder.build(); } @Override public TbQueueConsumer> getTransportNotificationsConsumer() { - return null; + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + nodeIdProvider.getNodeId()); + responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + return responseBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java index ef9cd23ac3..ba93bf96ef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java @@ -37,7 +37,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueProducer> getTransportMsgProducer(); + TbQueueProducer> getTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java index 2efa9ffbc6..b53c3ac28b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java @@ -35,7 +35,7 @@ public interface TbRuleEngineQueueProvider { * * @return */ - TbQueueProducer> getTransportMsgProducer(); + TbQueueProducer> getTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 19bf43f7db..f1b73f30af 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -119,6 +119,7 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + transportNotificationsConsumer.subscribe(); transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { while (!stopped) { From 94eb213716f0cb19364bb08d938c9f980872d2db Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Mar 2020 18:38:09 +0200 Subject: [PATCH 016/196] Implementation --- .../server/actors/ActorSystemContext.java | 6 +- .../device/DeviceActorMessageProcessor.java | 3 + .../server/actors/stats/StatsActor.java | 9 +- .../queue/DefaultTbCoreConsumerService.java | 24 +- .../server/service/queue/MsgPackCallback.java | 2 + .../service/queue/TbCoreConsumerStats.java | 5 + .../state/DefaultDeviceStateService.java | 299 +++++++++--------- .../service/state/DeviceStateService.java | 10 +- application/src/main/proto/cluster.proto | 10 - application/src/main/resources/logback.xml | 2 + .../common/AbstractTbQueueTemplate.java | 9 +- .../ConsistentHashPartitionService.java | 1 + ...veryService.java => DiscoveryService.java} | 2 +- .../discovery/DummyDiscoveryService.java | 2 +- ...ryService.java => ZkDiscoveryService.java} | 12 +- .../provider/KafkaMonolithQueueProvider.java | 22 +- .../proto/{transport.proto => queue.proto} | 15 + 17 files changed, 222 insertions(+), 211 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/discovery/{PartitionDiscoveryService.java => DiscoveryService.java} (93%) rename common/queue/src/main/java/org/thingsboard/server/discovery/{ZkPartitionDiscoveryService.java => ZkDiscoveryService.java} (95%) rename common/queue/src/main/proto/{transport.proto => queue.proto} (95%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 3db5c82ee2..6b8e950fdb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -361,7 +361,6 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.ERROR); - //TODO 2.5 event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); persistEvent(event); } @@ -371,7 +370,6 @@ public class ActorSystemContext { event.setTenantId(tenantId); event.setEntityId(entityId); event.setType(DataConstants.LC_EVENT); - //TODO 2.5 event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); persistEvent(event); } @@ -403,9 +401,7 @@ public class ActorSystemContext { public String getServerAddress() { - //TODO 2.5 -// return discoveryService.getCurrentServer().getServerAddress().toString(); - return null; + return serviceInfoProvider.getServiceId(); } public void persistDebugInput(TenantId tenantId, EntityId entityId, TbMsg tbMsg, String relationType) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6b878ef7d8..6b5dfbf6c3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -65,6 +65,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.queue.TbMsgCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; @@ -225,6 +226,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { boolean reportDeviceActivity = false; TransportToDeviceActorMsg msg = wrapper.getMsg(); + TbMsgCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } @@ -258,6 +260,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (reportDeviceActivity) { reportLogicalDeviceActivity(); } + callback.onSuccess(); } //TODO 2.5 move this as a notification to the queue; diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index c8e55bf9ed..8757deb2b4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -58,13 +58,12 @@ public class StatsActor extends ContextAwareActor { event.setEntityId(msg.getEntityId()); event.setTenantId(msg.getTenantId()); event.setType(DataConstants.STATS); - //TODO 2.5 -// event.setBody(toBodyJson(systemContext.getDiscoveryService().getCurrentServer().getServerAddress(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); + event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); systemContext.getEventService().save(event); } - private JsonNode toBodyJson(ServerAddress server, long messagesProcessed, long errorsOccurred) { - return mapper.createObjectNode().put("server", server.toString()).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { + return mapper.createObjectNode().put("server", serviceId).put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); } public static class ActorCreator extends ContextBasedCreator { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 1d3cf11f40..c95599f2ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,9 +29,11 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.discovery.PartitionChangeEvent; import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -59,14 +61,16 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private boolean statsEnabled; private final ActorSystemContext actorContext; + private final DeviceStateService stateService; private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext) { + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService) { this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); this.actorContext = actorContext; + this.stateService = stateService; } @PostConstruct @@ -88,7 +92,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { while (!stopped) { try { List> msgs = consumer.poll(pollDuration); - if(msgs.isEmpty()){ + if (msgs.isEmpty()) { continue; } ConcurrentMap> ackMap = msgs.stream().collect( @@ -98,11 +102,12 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { ToCoreMsg toCoreMsg = msg.getValue(); - log.trace("Forwarding message to rule engine {}", toCoreMsg); if (toCoreMsg.hasToDeviceActorMsg()) { + log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); - } else { - callback.onSuccess(); + } else if (toCoreMsg.hasDeviceStateServiceMsg()) { + log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); + forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); } } catch (Throwable e) { callback.onFailure(e); @@ -124,6 +129,13 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { }); } + private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { if (statsEnabled) { stats.log(toDeviceActorMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index 5b94db28b0..d7eebd98b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -36,6 +36,7 @@ public class MsgPackCallback i @Override public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); if (ackMap.remove(id) != null && ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } @@ -43,6 +44,7 @@ public class MsgPackCallback i @Override public void onFailure(Throwable t) { + log.trace("[{}] ON FAILURE", id); TbProtoQueueMsg message = ackMap.remove(id); log.warn("Failed to process message: {}", message.getValue(), t); if (ackMap.isEmpty()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index f7912836c5..f2956ea747 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -57,6 +57,10 @@ public class TbCoreConsumerStats { } } + public void log(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg) { + //TODO 2.5 + } + public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { @@ -67,4 +71,5 @@ public class TbCoreConsumerStats { subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 31b712644d..8d82b90fb6 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -33,6 +33,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -50,13 +51,19 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.discovery.PartitionService; +import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -67,7 +74,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -118,6 +124,12 @@ public class DefaultDeviceStateService implements DeviceStateService { @Lazy private ActorService actorService; + @Autowired + private TbCoreQueueProvider queueProvider; + + @Autowired + private PartitionService partitionService; + @Autowired private TelemetrySubscriptionService tsSubService; @@ -151,7 +163,6 @@ public class DefaultDeviceStateService implements DeviceStateService { queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); - //TODO: schedule persistence in v2.1; } @PreDestroy @@ -163,38 +174,79 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onDeviceAdded(Device device) { - queueExecutor.submit(() -> onDeviceAddedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), true, false, false); } @Override public void onDeviceUpdated(Device device) { - queueExecutor.submit(() -> onDeviceUpdatedSync(device)); + sendDeviceEvent(device.getTenantId(), device.getId(), false, true, false); + } + + @Override + public void onDeviceDeleted(Device device) { + sendDeviceEvent(device.getTenantId(), device.getId(), false, false, true); } @Override public void onDeviceConnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceConnectSync(deviceId)); + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastConnectTime(ts); + pushRuleEngineMessage(stateData, CONNECT_EVENT); + save(deviceId, LAST_CONNECT_TIME, ts); + } } @Override public void onDeviceActivity(DeviceId deviceId) { deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); - queueExecutor.submit(() -> onDeviceActivitySync(deviceId)); + long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); + long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); + if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + DeviceState state = stateData.getState(); + stateData.getState().setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); + save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); + deviceLastSavedActivity.put(deviceId, lastReportedActivity); + if (!state.isActive()) { + state.setActive(true); + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } + } } @Override public void onDeviceDisconnect(DeviceId deviceId) { - queueExecutor.submit(() -> onDeviceDisconnectSync(deviceId)); - } - - @Override - public void onDeviceDeleted(Device device) { - queueExecutor.submit(() -> onDeviceDeleted(device.getTenantId(), device.getId())); + DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + stateData.getState().setLastDisconnectTime(ts); + pushRuleEngineMessage(stateData, DISCONNECT_EVENT); + save(deviceId, LAST_DISCONNECT_TIME, ts); + } } @Override public void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - queueExecutor.submit(() -> onInactivityTimeoutUpdate(deviceId, inactivityTimeout)); + if (inactivityTimeout == 0L) { + return; + } + DeviceStateData stateData = deviceStates.get(deviceId); + if (stateData != null) { + long ts = System.currentTimeMillis(); + DeviceState state = stateData.getState(); + state.setInactivityTimeout(inactivityTimeout); + boolean oldActive = state.isActive(); + state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); + if (!oldActive && state.isActive() || oldActive && !state.isActive()) { + save(deviceId, ACTIVITY_STATE, state.isActive()); + } + } } @Override @@ -206,26 +258,56 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onRemoteMsg(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.DeviceStateServiceMsgProto proto; + public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbMsgCallback callback) { try { - proto = ClusterAPIProtos.DeviceStateServiceMsgProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); - if (proto.getDeleted()) { - queueExecutor.submit(() -> onDeviceDeleted(tenantId, deviceId)); - } else { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - if (proto.getAdded()) { - onDeviceAdded(device); - } else if (proto.getUpdated()) { - onDeviceUpdated(device); + TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); + if (proto.getDeleted()) { + onDeviceDeleted(tenantId, deviceId); + callback.onSuccess(); + } else { + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + if (proto.getAdded()) { + Futures.addCallback(fetchDeviceState(device), new FutureCallback() { + @Override + public void onSuccess(@Nullable DeviceStateData state) { + addDeviceUsingState(state); + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to register device to the state service", t); + callback.onFailure(t); + } + }); + } else if (proto.getUpdated()) { + DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); + if (stateData != null) { + TbMsgMetaData md = new TbMsgMetaData(); + md.putValue("deviceName", device.getName()); + md.putValue("deviceType", device.getType()); + stateData.setMetaData(md); + } + } + } else { + //Device was probably deleted while message was in queue; + callback.onSuccess(); } } + + } catch (Exception e) { + log.trace("Failed to process queue msg: [{}]", proto, e); + callback.onFailure(e); + } + + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + repartition(partitionChangeEvent.getPartitions()); } } @@ -241,9 +323,9 @@ public class DefaultDeviceStateService implements DeviceStateService { for (Device device : page.getData()) { //TODO 2.5 // if (!routingService.resolveById(device.getId()).isPresent()) { - if (!deviceStates.containsKey(device.getId())) { - fetchFutures.add(fetchDeviceState(device)); - } + if (!deviceStates.containsKey(device.getId())) { + fetchFutures.add(fetchDeviceState(device)); + } // } else { // Set tenantDeviceSet = tenantDevices.get(tenant.getId()); // if (tenantDeviceSet != null) { @@ -275,7 +357,7 @@ public class DefaultDeviceStateService implements DeviceStateService { for (Device device : page.getData()) { //TODO 2.5 // if (!routingService.resolveById(device.getId()).isPresent()) { - fetchFutures.add(fetchDeviceState(device)); + fetchFutures.add(fetchDeviceState(device)); // } } try { @@ -319,105 +401,29 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private void onDeviceConnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastConnectTime(ts); - pushRuleEngineMessage(stateData, CONNECT_EVENT); - save(deviceId, LAST_CONNECT_TIME, ts); - } - } - - private void onDeviceDisconnectSync(DeviceId deviceId) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - stateData.getState().setLastDisconnectTime(ts); - pushRuleEngineMessage(stateData, DISCONNECT_EVENT); - save(deviceId, LAST_DISCONNECT_TIME, ts); - } - } - - private void onDeviceActivitySync(DeviceId deviceId) { - long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); - long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); - if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { - DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); - if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); - save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); - deviceLastSavedActivity.put(deviceId, lastReportedActivity); - if (!state.isActive()) { - state.setActive(true); - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - } - private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { //TODO 2.5 // if (!routingService.resolveById(deviceId).isPresent()) { - Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); - if (device != null) { - try { - deviceStateData = fetchDeviceState(device).get(); - deviceStates.putIfAbsent(deviceId, deviceStateData); - } catch (InterruptedException | ExecutionException e) { - log.debug("[{}] Failed to fetch device state!", deviceId, e); - } + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + if (device != null) { + try { + deviceStateData = fetchDeviceState(device).get(); + deviceStates.putIfAbsent(deviceId, deviceStateData); + } catch (InterruptedException | ExecutionException e) { + log.debug("[{}] Failed to fetch device state!", deviceId, e); } + } // } } return deviceStateData; } - private void onInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout) { - if (inactivityTimeout == 0L) { - return; - } - DeviceStateData stateData = deviceStates.get(deviceId); - if (stateData != null) { - long ts = System.currentTimeMillis(); - DeviceState state = stateData.getState(); - state.setInactivityTimeout(inactivityTimeout); - boolean oldActive = state.isActive(); - state.setActive(ts < state.getLastActivityTime() + state.getInactivityTimeout()); - if (!oldActive && state.isActive() || oldActive && !state.isActive()) { - save(deviceId, ACTIVITY_STATE, state.isActive()); - } - } - } - - private void onDeviceAddedSync(Device device) { - //TODO 2.5 -// Optional address = routingService.resolveById(device.getId()); -// if (!address.isPresent()) { - Futures.addCallback(fetchDeviceState(device), new FutureCallback() { - @Override - public void onSuccess(@Nullable DeviceStateData state) { - addDeviceUsingState(state); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to register device to the state service", t); - } - }); -// } else { -// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), true, false, false); -// } - } - - private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, ServerAddress address, boolean added, boolean updated, boolean deleted) { - log.trace("[{}][{}] Device is monitored on other server: {}", tenantId, deviceId, address); - ClusterAPIProtos.DeviceStateServiceMsgProto.Builder builder = ClusterAPIProtos.DeviceStateServiceMsgProto.newBuilder(); + private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + log.trace("[{}][{}] Device is monitored on partition: {}", tenantId, deviceId, tpi); + TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setDeviceIdMSB(deviceId.getId().getMostSignificantBits()); @@ -425,43 +431,22 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - //TODO 2.5 -// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_DEVICE_STATE_SERVICE_MESSAGE, builder.build().toByteArray()); - } - - private void onDeviceUpdatedSync(Device device) { - //TODO 2.5 -// Optional address = routingService.resolveById(device.getId()); -// if (!address.isPresent()) { - DeviceStateData stateData = getOrFetchDeviceStateData(device.getId()); - if (stateData != null) { - TbMsgMetaData md = new TbMsgMetaData(); - md.putValue("deviceName", device.getName()); - md.putValue("deviceType", device.getType()); - stateData.setMetaData(md); - } -// } else { -// sendDeviceEvent(device.getTenantId(), device.getId(), address.get(), false, true, false); -// } + TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); + queueProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), + TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build()), null); } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { - //TODO 2.5 -// Optional address = routingService.resolveById(deviceId); -// if (!address.isPresent()) { - deviceStates.remove(deviceId); - deviceLastReportedActivity.remove(deviceId); - deviceLastSavedActivity.remove(deviceId); - Set deviceIds = tenantDevices.get(tenantId); - if (deviceIds != null) { - deviceIds.remove(deviceId); - if (deviceIds.isEmpty()) { - tenantDevices.remove(tenantId); - } + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + Set deviceIds = tenantDevices.get(tenantId); + if (deviceIds != null) { + deviceIds.remove(deviceId); + if (deviceIds.isEmpty()) { + tenantDevices.remove(tenantId); } -// } else { -// sendDeviceEvent(tenantId, deviceId, address.get(), false, false, true); -// } + } } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 8dbe7579e3..636a9e9bb1 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.service.state; +import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.service.queue.TbMsgCallback; /** * Created by ashvayka on 01.05.18. */ -public interface DeviceStateService { +public interface DeviceStateService extends ApplicationListener { void onDeviceAdded(Device device); @@ -38,7 +41,6 @@ public interface DeviceStateService { void onDeviceInactivityTimeoutUpdate(DeviceId deviceId, long inactivityTimeout); - void onClusterUpdate(); + void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto serverAddress, TbMsgCallback bytes); - void onRemoteMsg(ServerAddress serverAddress, byte[] bytes); } diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto index cfacc66121..46526e8e1c 100644 --- a/application/src/main/proto/cluster.proto +++ b/application/src/main/proto/cluster.proto @@ -134,13 +134,3 @@ message FromDeviceRPCResponseProto { string response = 3; int32 error = 4; } - -message DeviceStateServiceMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 deviceIdMSB = 3; - int64 deviceIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e147325f42..3d47db432f 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java index b70fc417c8..a6193a0b48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -46,16 +46,13 @@ public class AbstractTbQueueTemplate { return new String(data, StandardCharsets.UTF_8); } - private static ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); - protected static byte[] longToBytes(long x) { + ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } protected static long bytesToLong(byte[] bytes) { - longBuffer.put(bytes, 0, bytes.length); - longBuffer.flip();//need flip - return longBuffer.getLong(); + return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java index 25c936f829..831748da06 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java @@ -101,6 +101,7 @@ public class ConsistentHashPartitionService implements PartitionService { return topicPartitions; } + //TODO 2.5 This should return cached TopicPartitionInfo objects instead of creating new one every time. @Override public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java index 6fe4571517..8272df5fdc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java @@ -15,6 +15,6 @@ */ package org.thingsboard.server.discovery; -public interface PartitionDiscoveryService { +public interface DiscoveryService { } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java index 8983cf1e97..28fd5c1fb3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java @@ -28,7 +28,7 @@ import java.util.Collections; @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @Slf4j @DependsOn("environmentLogService") -public class DummyDiscoveryService implements PartitionDiscoveryService { +public class DummyDiscoveryService implements DiscoveryService { private final TbServiceInfoProvider serviceInfoProvider; private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java index 07f11617ac..48c886b7e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkPartitionDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.discovery; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -31,7 +30,6 @@ import org.apache.curator.retry.RetryForever; import org.apache.curator.utils.CloseableUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -39,18 +37,12 @@ import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.transport.TransportProtos; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -60,7 +52,7 @@ import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent. @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false) @Slf4j -public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, PathChildrenCacheListener { +public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener { @Value("${zk.url}") private String zkUrl; @@ -84,7 +76,7 @@ public class ZkPartitionDiscoveryService implements PartitionDiscoveryService, P private volatile boolean stopped = true; - public ZkPartitionDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { this.serviceInfoProvider = serviceInfoProvider; this.partitionService = partitionService; } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java index 01e7a211d3..b7412a7c99 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private TbQueueProducer> tbCoreProducer; + public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings, @@ -77,13 +79,21 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return requestBuilder.build(); } + //TODO 2.5 Singleton @Override public TbQueueProducer> getTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - return requestBuilder.build(); + if (tbCoreProducer == null) { + synchronized (this) { + if (tbCoreProducer == null) { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + tbCoreProducer = requestBuilder.build(); + } + } + } + return tbCoreProducer; } @Override diff --git a/common/queue/src/main/proto/transport.proto b/common/queue/src/main/proto/queue.proto similarity index 95% rename from common/queue/src/main/proto/transport.proto rename to common/queue/src/main/proto/queue.proto index 8c564b6540..17f8b9c31c 100644 --- a/common/queue/src/main/proto/transport.proto +++ b/common/queue/src/main/proto/queue.proto @@ -239,6 +239,20 @@ message DeviceActorToTransportMsg { ToServerRpcResponseMsg toServerResponse = 7; } +/** + * TB Core to TB Core messages + */ + +message DeviceStateServiceMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + /** * Main messages; */ @@ -259,6 +273,7 @@ message TransportApiResponseMsg { /* Messages that are handled by ThingsBoard Core Service */ message ToCoreMsg { TransportToDeviceActorMsg toDeviceActorMsg = 1; + DeviceStateServiceMsgProto deviceStateServiceMsg = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From 7203e7aa19a208a850c45149f14c0be348b53677 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 Mar 2020 18:07:24 +0200 Subject: [PATCH 017/196] Distributed Mode fixes --- .../server/actors/ActorSystemContext.java | 9 +- .../actors/ruleChain/DefaultTbContext.java | 2 +- .../server/actors/stats/StatsActor.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 12 +- .../DefaultTbRuleEngineConsumerService.java | 10 +- .../server/service/queue/MsgPackCallback.java | 2 +- .../service/queue/TbCoreConsumerService.java | 5 +- .../queue/TbRuleEngineConsumerService.java | 2 +- .../service/script/RemoteJsInvokeService.java | 6 +- .../script/RemoteJsRequestEncoder.java | 4 +- .../script/RemoteJsResponseDecoder.java | 6 +- .../state/DefaultDeviceStateService.java | 129 +++++----- .../service/state/DeviceStateService.java | 2 +- .../DefaultTbCoreToTransportService.java | 12 +- .../transport/DefaultTransportApiService.java | 2 +- .../transport/RemoteTransportApiService.java | 18 +- .../transport/ToRuleEngineMsgDecoder.java | 4 +- .../transport/ToTransportMsgEncoder.java | 2 +- .../transport/TransportApiService.java | 4 +- .../ConsistentHashParitionServiceTest.java | 8 +- .../server/discovery/TopicPartitionInfo.java | 41 ---- .../server/kafka/TbKafkaProperty.java | 32 --- .../server/kafka/TbNodeIdProvider.java | 51 ---- .../server/{ => queue}/TbQueueAdmin.java | 2 +- .../server/{ => queue}/TbQueueCallback.java | 2 +- .../server/{ => queue}/TbQueueConsumer.java | 7 +- .../{ => queue}/TbQueueCoreSettings.java | 2 +- .../server/{ => queue}/TbQueueHandler.java | 2 +- .../server/{ => queue}/TbQueueMsg.java | 2 +- .../server/{ => queue}/TbQueueMsgHeaders.java | 2 +- .../{ => queue}/TbQueueMsgMetadata.java | 2 +- .../server/{ => queue}/TbQueueProducer.java | 4 +- .../{ => queue}/TbQueueRequestTemplate.java | 2 +- .../{ => queue}/TbQueueResponseTemplate.java | 2 +- .../TbQueueRuleEngineSettings.java | 2 +- .../TbQueueTransportApiSettings.java | 2 +- .../TbQueueTransportNotificationSettings.java | 2 +- .../common/AbstractTbQueueTemplate.java | 4 +- .../common/AsyncCallbackTemplate.java | 2 +- .../common/DefaultTbQueueMsgHeaders.java | 4 +- .../common/DefaultTbQueueRequestTemplate.java | 18 +- .../DefaultTbQueueResponseTemplate.java | 14 +- .../{ => queue}/common/TbProtoQueueMsg.java | 6 +- .../discovery/ConsistentHashCircle.java | 2 +- .../ConsistentHashPartitionService.java | 20 +- .../DefaultTbServiceInfoProvider.java | 4 +- .../discovery/DiscoveryService.java | 2 +- .../discovery/DummyDiscoveryService.java | 2 +- .../discovery/PartitionChangeEvent.java | 9 +- .../discovery/PartitionService.java | 10 +- .../{ => queue}/discovery/ServiceKey.java | 2 +- .../{ => queue}/discovery/ServiceType.java | 8 +- .../discovery/TbServiceInfoProvider.java | 2 +- .../queue/discovery/TopicPartitionInfo.java | 78 ++++++ .../discovery/ZkDiscoveryService.java | 2 +- .../environment/EnvironmentLogService.java | 2 +- .../{ => queue}/kafka/KafkaTbQueueMsg.java | 8 +- .../kafka/KafkaTbQueueMsgMetadata.java | 4 +- .../{ => queue}/kafka/TBKafkaAdmin.java | 7 +- .../kafka/TBKafkaConsumerTemplate.java | 14 +- .../kafka/TBKafkaProducerTemplate.java | 26 +- .../{ => queue}/kafka/TbKafkaDecoder.java | 4 +- .../{ => queue}/kafka/TbKafkaEncoder.java | 2 +- .../{ => queue}/kafka/TbKafkaHandler.java | 2 +- .../{ => queue}/kafka/TbKafkaPartitioner.java | 2 +- .../server/queue/kafka/TbKafkaProperty.java | 28 +++ .../{ => queue}/kafka/TbKafkaSettings.java | 3 +- .../{ => queue}/memory/InMemoryStorage.java | 6 +- .../memory/InMemoryTbQueueConsumer.java | 10 +- .../memory/InMemoryTbQueueProducer.java | 10 +- .../InMemoryMonolithQueueProvider.java | 21 +- .../InMemoryTransportQueueProvider.java | 28 +-- .../provider/KafkaMonolithQueueProvider.java | 52 ++-- .../provider/KafkaTbCoreQueueProvider.java | 82 ++++--- .../KafkaTbRuleEngineQueueProvider.java | 63 ++--- .../provider/KafkaTransportQueueProvider.java | 58 ++--- .../provider/TbCoreQueueProvider.java | 10 +- .../provider/TbRuleEngineQueueProvider.java | 10 +- .../provider/TransportQueueProvider.java | 10 +- .../transport/coap/CoapTransportContext.java | 2 +- .../transport/coap/CoapTransportService.java | 2 +- .../transport/http/DeviceApiController.java | 2 +- .../transport/http/HttpTransportContext.java | 2 +- .../transport/mqtt/MqttTransportContext.java | 12 +- .../transport/mqtt/MqttTransportService.java | 2 +- .../common/transport/TransportContext.java | 13 +- .../service/DefaultTransportService.java | 25 +- .../service/ToRuleEngineMsgEncoder.java | 2 +- .../ToTransportMsgResponseDecoder.java | 4 +- .../service/TransportApiRequestEncoder.java | 2 +- .../service/TransportApiResponseDecoder.java | 4 +- .../ThingsboardCoapTransportApplication.java | 2 +- .../ThingsboardHttpTransportApplication.java | 2 +- .../ThingsboardMqttTransportApplication.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 36 ++- ui/package-lock.json | 225 ++++++++---------- 96 files changed, 690 insertions(+), 700 deletions(-) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueAdmin.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueCallback.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueConsumer.java (82%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueCoreSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueHandler.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueMsg.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueMsgHeaders.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueMsgMetadata.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueProducer.java (88%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueRequestTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueResponseTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueRuleEngineSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueTransportApiSettings.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/TbQueueTransportNotificationSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/AbstractTbQueueTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/AsyncCallbackTemplate.java (98%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/DefaultTbQueueMsgHeaders.java (91%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/DefaultTbQueueRequestTemplate.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/DefaultTbQueueResponseTemplate.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/common/TbProtoQueueMsg.java (90%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ConsistentHashCircle.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ConsistentHashPartitionService.java (92%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/DefaultTbServiceInfoProvider.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/DiscoveryService.java (93%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/DummyDiscoveryService.java (97%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/PartitionChangeEvent.java (85%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/PartitionService.java (68%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ServiceKey.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ServiceType.java (72%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/TbServiceInfoProvider.java (94%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/discovery/ZkDiscoveryService.java (99%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/environment/EnvironmentLogService.java (96%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/KafkaTbQueueMsg.java (87%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/KafkaTbQueueMsgMetadata.java (89%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TBKafkaAdmin.java (93%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TBKafkaConsumerTemplate.java (90%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TBKafkaProducerTemplate.java (76%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaDecoder.java (89%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaEncoder.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaHandler.java (94%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaPartitioner.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/kafka/TbKafkaSettings.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/memory/InMemoryStorage.java (95%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/memory/InMemoryTbQueueConsumer.java (81%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/memory/InMemoryTbQueueProducer.java (84%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/InMemoryMonolithQueueProvider.java (85%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/InMemoryTransportQueueProvider.java (81%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaMonolithQueueProvider.java (75%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaTbCoreQueueProvider.java (52%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaTbRuleEngineQueueProvider.java (53%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/KafkaTransportQueueProvider.java (72%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/TbCoreQueueProvider.java (88%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/TbRuleEngineQueueProvider.java (83%) rename common/queue/src/main/java/org/thingsboard/server/{ => queue}/provider/TransportQueueProvider.java (84%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 6b8e950fdb..46e6df60ab 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -35,7 +35,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; @@ -46,7 +45,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; @@ -65,8 +63,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.discovery.TbServiceInfoProvider; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -321,10 +318,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Autowired - @Getter - private TbNodeIdProvider nodeIdProvider; - @Getter @Setter private ActorRef appActor; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 3e982b520e..2de491c89d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -254,7 +254,7 @@ class DefaultTbContext implements TbContext { @Override public String getNodeId() { - return mainCtx.getNodeIdProvider().getNodeId(); + return mainCtx.getServiceInfoProvider().getServiceId(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 8757deb2b4..1ef33467b2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c95599f2ad..07b7704d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,15 +24,15 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 253c7d9933..6d06b9ee11 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -23,13 +23,13 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.ServiceType; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index d7eebd98b0..d275adc4be 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; import java.util.concurrent.ConcurrentMap; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 8a39b5df4a..2dcdb0e0c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -16,10 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.function.Consumer; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; public interface TbCoreConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index 968fbb2274..19966b4566 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; public interface TbRuleEngineConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 08578c3890..8150a77c30 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -24,9 +24,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; import javax.annotation.Nullable; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java index 1b33fd85cc..51ecba27b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java @@ -17,9 +17,9 @@ package org.thingsboard.server.service.script; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; import java.nio.charset.StandardCharsets; diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java index ef15d5c877..621d2b05d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java @@ -16,10 +16,10 @@ package org.thingsboard.server.service.script; import com.google.protobuf.util.JsonFormat; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 8d82b90fb6..4e395f7090 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +32,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -56,13 +55,12 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.discovery.PartitionChangeEvent; -import org.thingsboard.server.discovery.PartitionService; -import org.thingsboard.server.discovery.ServiceType; -import org.thingsboard.server.discovery.TopicPartitionInfo; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -153,6 +151,7 @@ public class DefaultDeviceStateService implements DeviceStateService { private ListeningScheduledExecutorService queueExecutor; private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); + private ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); @@ -161,7 +160,6 @@ public class DefaultDeviceStateService implements DeviceStateService { public void init() { // Should be always single threaded due to absence of locks. queueExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-state"))); - queueExecutor.submit(this::initStateFromDB); queueExecutor.scheduleAtFixedRate(this::updateState, new Random().nextInt(defaultStateCheckIntervalInSec), defaultStateCheckIntervalInSec, TimeUnit.SECONDS); } @@ -249,14 +247,6 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - @Override - public void onClusterUpdate() { - if (!clusterUpdatePending) { - clusterUpdatePending = true; - queueExecutor.submit(this::onClusterUpdateSync); - } - } - @Override public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbMsgCallback callback) { try { @@ -272,8 +262,15 @@ public class DefaultDeviceStateService implements DeviceStateService { Futures.addCallback(fetchDeviceState(device), new FutureCallback() { @Override public void onSuccess(@Nullable DeviceStateData state) { - addDeviceUsingState(state); - callback.onSuccess(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, device.getId()); + if (partitionedDevices.containsKey(tpi)) { + addDeviceUsingState(tpi, state); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Device belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , tenantId, deviceId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Device belongs to external partition " + tpi.getFullTopicName() + "!")); + } } @Override @@ -307,61 +304,63 @@ public class DefaultDeviceStateService implements DeviceStateService { @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { - repartition(partitionChangeEvent.getPartitions()); - } - } - - private void onClusterUpdateSync() { - clusterUpdatePending = false; - List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); - for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); - TextPageLink pageLink = new TextPageLink(initFetchPackSize); - while (pageLink != null) { - TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); - pageLink = page.getNextPageLink(); - for (Device device : page.getData()) { - //TODO 2.5 -// if (!routingService.resolveById(device.getId()).isPresent()) { - if (!deviceStates.containsKey(device.getId())) { - fetchFutures.add(fetchDeviceState(device)); - } -// } else { -// Set tenantDeviceSet = tenantDevices.get(tenant.getId()); -// if (tenantDeviceSet != null) { -// tenantDeviceSet.remove(device.getId()); -// } -// deviceStates.remove(device.getId()); -// deviceLastReportedActivity.remove(device.getId()); -// deviceLastSavedActivity.remove(device.getId()); -// } - } - try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); - } catch (InterruptedException | ExecutionException e) { - log.warn("Failed to init device state service from DB", e); + synchronized (this) { + if (!clusterUpdatePending) { + clusterUpdatePending = true; + queueExecutor.submit(() -> { + clusterUpdatePending = false; + initStateFromDB(partitionChangeEvent.getPartitions()); + }); } } } } - private void initStateFromDB() { + private void initStateFromDB(Set partitions) { try { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(partitionedDevices.keySet()); + + Set removedPartitions = new HashSet<>(partitionedDevices.keySet()); + removedPartitions.removeAll(partitions); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set devices = partitionedDevices.remove(partition); + devices.forEach(deviceId -> { + deviceStates.remove(deviceId); + deviceLastReportedActivity.remove(deviceId); + deviceLastSavedActivity.remove(deviceId); + }); + }); + + //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. + // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); for (Tenant tenant : tenants) { - List> fetchFutures = new ArrayList<>(); TextPageLink pageLink = new TextPageLink(initFetchPackSize); while (pageLink != null) { + List> fetchFutures = new ArrayList<>(); TextPageData page = deviceService.findDevicesByTenantId(tenant.getId(), pageLink); pageLink = page.getNextPageLink(); for (Device device : page.getData()) { - //TODO 2.5 -// if (!routingService.resolveById(device.getId()).isPresent()) { - fetchFutures.add(fetchDeviceState(device)); -// } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), device.getId()); + if (addedPartitions.contains(tpi)) { + ListenableFuture future = Futures.transform(fetchDeviceState(device), new Function() { + @Nullable + @Override + public Void apply(@Nullable DeviceStateData state) { + if (state != null) { + addDeviceUsingState(tpi, state); + } + return null; + } + }); + fetchFutures.add(future); + } } try { - Futures.successfulAsList(fetchFutures).get().forEach(this::addDeviceUsingState); + Futures.successfulAsList(fetchFutures).get(); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to init device state service from DB", e); } @@ -372,11 +371,12 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private void addDeviceUsingState(DeviceStateData state) { - tenantDevices.computeIfAbsent(state.getTenantId(), id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); + private void addDeviceUsingState(TopicPartitionInfo tpi, DeviceStateData state) { + partitionedDevices.computeIfAbsent(tpi, id -> ConcurrentHashMap.newKeySet()).add(state.getDeviceId()); deviceStates.put(state.getDeviceId(), state); } + //TODO 2.5: review this method private void updateState() { long ts = System.currentTimeMillis(); Set deviceIds = new HashSet<>(deviceStates.keySet()); @@ -404,8 +404,6 @@ public class DefaultDeviceStateService implements DeviceStateService { private DeviceStateData getOrFetchDeviceStateData(DeviceId deviceId) { DeviceStateData deviceStateData = deviceStates.get(deviceId); if (deviceStateData == null) { - //TODO 2.5 -// if (!routingService.resolveById(deviceId).isPresent()) { Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); if (device != null) { try { @@ -415,7 +413,6 @@ public class DefaultDeviceStateService implements DeviceStateService { log.debug("[{}] Failed to fetch device state!", deviceId, e); } } -// } } return deviceStateData; } @@ -538,7 +535,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } } - private class AttributeSaveCallback implements FutureCallback { + private static class AttributeSaveCallback implements FutureCallback { private final DeviceId deviceId; private final String key; private final Object value; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 636a9e9bb1..14a25c2e9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.state; import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.queue.TbMsgCallback; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index 833b28c3b2..fb369f4564 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -19,14 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import java.util.UUID; import java.util.function.Consumer; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 51597877b4..42ee4c4619 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java index e84e1b595e..4c9c801e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java @@ -16,24 +16,18 @@ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueResponseTemplate; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.DefaultTbQueueResponseTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TbNodeIdProvider; -import org.thingsboard.server.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java index bdebacbd13..71739c0439 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java index 9c3f5634bf..50b940ae48 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.transport; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java index eab2e07b0e..c9edfa9331 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TransportApiService.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.TbQueueHandler; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 9442202688..445465f5df 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -25,10 +25,10 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.discovery.ConsistentHashPartitionService; -import org.thingsboard.server.discovery.ServiceType; -import org.thingsboard.server.discovery.TbServiceInfoProvider; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.ArrayList; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java deleted file mode 100644 index 4480d07dd6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TopicPartitionInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.discovery; - -import lombok.Builder; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Optional; - -@Builder -public class TopicPartitionInfo { - - private final String topic; - private final TenantId tenantId; - private final Integer partition; - - public String getTopic() { - return topic; - } - - public Optional getTenantId() { - return Optional.ofNullable(tenantId); - } - - public Optional getPartition() { - return Optional.ofNullable(partition); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java deleted file mode 100644 index cd52948da8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaProperty.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -/** - * Created by ashvayka on 25.09.18. - */ -@Data -public class TbKafkaProperty { - - private String key; - private String value; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java b/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java deleted file mode 100644 index 343c5e6dd8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbNodeIdProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.kafka; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.net.UnknownHostException; - -/** - * Created by ashvayka on 12.10.18. - */ -@Slf4j -@Component -public class TbNodeIdProvider { - - @Getter - @Value("${cluster.node_id:#{null}}") - private String nodeId; - - @PostConstruct - public void init() { - if (StringUtils.isEmpty(nodeId)) { - try { - nodeId = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - nodeId = org.apache.commons.lang3.RandomStringUtils.randomAlphabetic(10); - } - } - log.info("Current NodeId: {}", nodeId); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index 5952e7da70..deb84f440e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueAdmin { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index e823d2a5fc..f21ad80d10 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCallback.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueCallback { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java similarity index 82% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index ac4d7611f6..1a5c98ec80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -13,9 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; + +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; +import java.util.Set; public interface TbQueueConsumer { @@ -23,7 +26,7 @@ public interface TbQueueConsumer { void subscribe(); - void subscribe(List partitions); + void subscribe(Set partitions); void unsubscribe(); diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java index dc84625634..4957b5ed5b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java index 93004cddca..609cbd2107 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java index ca8f3a61da..b0e4990305 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java index f95454c976..f205ed676f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgHeaders.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java index a3331999a5..0c24fec438 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgMetadata.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueMsgMetadata { } diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java similarity index 88% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index 2498117c46..e7ea35b836 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; public interface TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index 2eb429b082..20530360d6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java index 686570ca32..c823b3df88 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; public interface TbQueueResponseTemplate { diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java index 4acfec1091..fc3a36a88a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java index b98bd3b76d..263cf262a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java index 2666045e7e..0f2ac55862 100644 --- a/common/queue/src/main/java/org/thingsboard/server/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server; +package org.thingsboard.server.queue; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index a6193a0b48..ec3b0b537a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java similarity index 98% rename from common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java index 95783f87cd..d06645abd3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/AsyncCallbackTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java similarity index 91% rename from common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java index de0c368a65..36c3dd0999 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueMsgHeaders.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsgHeaders.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; -import org.thingsboard.server.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.HashMap; import java.util.Map; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 0388daebb6..a2e92e112b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -21,14 +21,14 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.errors.InterruptException; -import org.thingsboard.server.TbQueueAdmin; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index c77692ee36..3e70040e2a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.errors.InterruptException; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueHandler; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueResponseTemplate; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java index 8dd492504c..8823002530 100644 --- a/common/queue/src/main/java/org/thingsboard/server/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common; +package org.thingsboard.server.queue.common; import lombok.Data; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java index b4c382050b..1b40545cde 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashCircle.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashCircle.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java similarity index 92% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 831748da06..97b660a48c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import com.google.common.hash.HashCode; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -40,6 +39,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; +import java.util.stream.Collectors; @Service @Slf4j @@ -90,6 +90,7 @@ public class ConsistentHashPartitionService implements PartitionService { List partitions = myPartitions.get(serviceKey); List topicPartitions = new ArrayList<>(); for (Integer partition : partitions) { + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); @@ -104,11 +105,19 @@ public class ConsistentHashPartitionService implements PartitionService { //TODO 2.5 This should return cached TopicPartitionInfo objects instead of creating new one every time. @Override public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { - boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); int hash = hashFunction.newHasher() .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); int partition = Math.abs(hash % partitionSizes.get(serviceType)); + return buildTopicPartitionInfo(serviceType, tenantId, partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { + return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); @@ -153,7 +162,10 @@ public class ConsistentHashPartitionService implements PartitionService { myPartitions.forEach((serviceKey, partitions) -> { if (!partitions.equals(oldPartitions.get(serviceKey))) { log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); - applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, partitions)); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) + .collect(Collectors.toSet()); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } }); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 80040a017b..cb7b04df44 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -64,7 +64,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (serviceType.equalsIgnoreCase("monolith")) { serviceTypes = Collections.unmodifiableList(Arrays.asList(ServiceType.values())); } else { - serviceTypes = Collections.singletonList(ServiceType.valueOf(serviceType)); + serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); } ServiceInfo.Builder builder = ServiceInfo.newBuilder() .setServiceId(serviceId) diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java index 8272df5fdc..36510261d6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; public interface DiscoveryService { diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 28fd5c1fb3..9017823efb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index 570a774c9c..13976232e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.Getter; -import lombok.Setter; import org.springframework.context.ApplicationEvent; -import java.util.List; +import java.util.Set; public class PartitionChangeEvent extends ApplicationEvent { @@ -27,9 +26,9 @@ public class PartitionChangeEvent extends ApplicationEvent { @Getter private final ServiceKey serviceKey; @Getter - private final List partitions; + private final Set partitions; - public PartitionChangeEvent(Object source, ServiceKey serviceKey, List partitions) { + public PartitionChangeEvent(Object source, ServiceKey serviceKey, Set partitions) { super(source); this.serviceKey = serviceKey; this.partitions = partitions; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java similarity index 68% rename from common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index ad61a63807..f8613f5fb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -21,11 +21,19 @@ import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; +/** + * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} + */ public interface PartitionService { List getCurrentPartitions(ServiceType serviceType); TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + /** + * Received from the Discovery service when network topology is changed. + * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} + */ void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java index 0179be968c..3396078124 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java similarity index 72% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java index 801069eb61..a5de33c436 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ServiceType.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java @@ -13,8 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + + public static ServiceType of(String serviceType) { + return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 5cb545c5d6..764f646a8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java new file mode 100644 index 0000000000..05c5c1586d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Builder; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; +import java.util.Optional; + +public class TopicPartitionInfo { + + private final String topic; + private final TenantId tenantId; + private final Integer partition; + private final String fullTopicName; + + @Builder + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition) { + this.topic = topic; + this.tenantId = tenantId; + this.partition = partition; + String tmp = topic; + if (tenantId != null) { + tmp += "." + tenantId.getId().toString(); + } + if (partition != null) { + tmp += "." + partition; + } + + this.fullTopicName = tmp; + } + + public String getTopic() { + return topic; + } + + public Optional getTenantId() { + return Optional.ofNullable(tenantId); + } + + public Optional getPartition() { + return Optional.ofNullable(partition); + } + + public String getFullTopicName() { + return fullTopicName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfo that = (TopicPartitionInfo) o; + return topic.equals(that.topic) && + Objects.equals(tenantId, that.tenantId) && + Objects.equals(partition, that.partition) && + fullTopicName.equals(that.fullTopicName); + } + + @Override + public int hashCode() { + return Objects.hash(fullTopicName); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java similarity index 99% rename from common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 48c886b7e4..e761a229c4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.discovery; +package org.thingsboard.server.queue.discovery; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; diff --git a/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java index cff3fd1056..267b34b204 100644 --- a/common/queue/src/main/java/org/thingsboard/server/environment/EnvironmentLogService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/EnvironmentLogService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.environment; +package org.thingsboard.server.queue.environment; import lombok.extern.slf4j.Slf4j; import org.apache.zookeeper.Environment; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java similarity index 87% rename from common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index 9f8e73bdfc..98470eafba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueMsgHeaders; -import org.thingsboard.server.common.DefaultTbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java similarity index 89% rename from common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java index 09cc292deb..5a9eaa632c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/KafkaTbQueueMsgMetadata.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsgMetadata.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.AllArgsConstructor; import lombok.Data; import org.apache.kafka.clients.producer.RecordMetadata; -import org.thingsboard.server.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueMsgMetadata; @Data @AllArgsConstructor diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java index 5f1b41ea7b..e1ca580c28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java @@ -13,11 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.JdkFutureAdapters; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; @@ -25,7 +22,7 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.errors.TopicExistsException; -import org.thingsboard.server.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueAdmin; import java.util.Collections; import java.util.concurrent.ExecutionException; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index eb35c583c8..b4b1578939 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.Getter; @@ -22,8 +22,9 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.io.IOException; import java.time.Duration; @@ -31,7 +32,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.concurrent.ExecutionException; +import java.util.Set; import java.util.stream.Collectors; /** @@ -79,9 +80,10 @@ public class TBKafkaConsumerTemplate implements TbQueueCon } @Override - public void subscribe(List partitions) { - List topicNames = partitions.stream().map(partition -> topic + "." + partition).collect(Collectors.toList()); + public void subscribe(Set partitions) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); topicNames.forEach(this::createTopicIfNotExists); + consumer.unsubscribe(); consumer.subscribe(topicNames); subscribed = true; } diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java similarity index 76% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 4999ba026a..8f177ecb3f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.Getter; @@ -21,22 +21,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import java.util.List; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -76,14 +69,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro byte[] data = msg.getData(); ProducerRecord record; Iterable
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); - StringBuilder topic = new StringBuilder().append(tpi.getTopic()); - if (tpi.getTenantId().isPresent()) { - topic.append(".").append(tpi.getTenantId().get().getId().toString()); - } - if (tpi.getPartition().isPresent()) { - topic.append(".").append(tpi.getPartition().get()); - } - record = new ProducerRecord<>(topic.toString(), null, key, data, headers); + record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { if (callback != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java similarity index 89% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java index 564d905675..6e3ea1e4a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaDecoder.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import java.io.IOException; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java index 22f65c58a0..b3c4dec8ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaEncoder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; /** * Created by ashvayka on 25.09.18. diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java index 5e5aa0591a..8c2cbd2771 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaHandler.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import com.google.common.util.concurrent.ListenableFuture; diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java index 8ec5cd1d97..b31f87642b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaPartitioner.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import org.apache.kafka.clients.producer.Partitioner; import org.apache.kafka.common.PartitionInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java new file mode 100644 index 0000000000..b6de7db50d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProperty.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Data; + +/** + * Created by ashvayka on 25.09.18. + */ +@Data +public class TbKafkaProperty { + + private String key; + private String value; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 912e935bef..e8581d9719 100644 --- a/common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.kafka; +package org.thingsboard.server.queue.kafka; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java rename to common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index 86d103ea70..a61bf560ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -13,17 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.memory; +package org.thingsboard.server.queue.memory; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 3f1e844efc..c0849d083e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.memory; +package org.thingsboard.server.queue.memory; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import java.util.List; +import java.util.Set; public class InMemoryTbQueueConsumer implements TbQueueConsumer { private final InMemoryStorage storage = InMemoryStorage.getInstance(); @@ -40,7 +42,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon } @Override - public void subscribe(List partitions) { + public void subscribe(Set partitions) { } diff --git a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java similarity index 84% rename from common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java rename to common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index 79fa1d6b39..c235f876ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.memory; +package org.thingsboard.server.queue.memory; import lombok.Data; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueMsg; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java index 7fc7bdea7c..b734562a61 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java @@ -13,26 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Slf4j @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java similarity index 81% rename from common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java index 2d7205e516..3e33e4daa6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/InMemoryTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java @@ -13,33 +13,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueAdmin; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.memory.InMemoryTbQueueConsumer; -import org.thingsboard.server.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Component -@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport') && '${queue.type:null}'=='in-memory'") +@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class InMemoryTransportQueueProvider implements TransportQueueProvider { diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index b7412a7c99..990baca5a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,33 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; @@ -48,13 +48,13 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private TbQueueProducer> tbCoreProducer; public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, - TbNodeIdProvider nodeIdProvider, + TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -65,7 +65,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-notifications" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); return requestBuilder.build(); } @@ -74,7 +74,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); return requestBuilder.build(); } @@ -87,7 +87,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn if (tbCoreProducer == null) { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); tbCoreProducer = requestBuilder.build(); } @@ -101,8 +101,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @@ -112,8 +112,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-core-consumer-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @@ -123,8 +123,8 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); - consumerBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("transport-api-consumer-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @@ -133,7 +133,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-producer-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java similarity index 52% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 72338bb149..0d6d62c3cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -13,43 +13,59 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + private TbQueueProducer> tbCoreProducer; + + public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,18 +74,25 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @Override public TbQueueProducer> getTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - return requestBuilder.build(); + if (tbCoreProducer == null) { + synchronized (this) { + if (tbCoreProducer == null) { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + tbCoreProducer = requestBuilder.build(); + } + } + } + return tbCoreProducer; } @Override @@ -77,29 +100,28 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); - consumerBuilder.clientId("tb-core-consumer" + nodeIdProvider.getNodeId()); - consumerBuilder.groupId("tb-core-node-" + nodeIdProvider.getNodeId()); + consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override public TbQueueConsumer> getTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(transportApiSettings.getRequestsTopic()); + consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } @Override public TbQueueProducer> getTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java similarity index 53% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index d635419141..c75437f38b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -13,43 +13,54 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, TbNodeIdProvider nodeIdProvider, TbQueueCoreSettings coreSettings) { + public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; } - @Override public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -58,29 +69,27 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } - @Override public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-core-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); - responseBuilder.settings(kafkaSettings); - responseBuilder.topic(coreSettings.getTopic()); - responseBuilder.clientId("tb-rule-engine-consumer-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("tb-rule-engine-" + nodeIdProvider.getNodeId()); - responseBuilder.autoCommit(true); - responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - return responseBuilder.build(); + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(ruleEngineSettings.getTopic()); + consumerBuilder.clientId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java similarity index 72% rename from common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java index 577c9180fc..10cc9c1871 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java @@ -13,30 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueCoreSettings; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.TbQueueRuleEngineSettings; -import org.thingsboard.server.TbQueueTransportApiSettings; -import org.thingsboard.server.TbQueueTransportNotificationSettings; -import org.thingsboard.server.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TBKafkaAdmin; -import org.thingsboard.server.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.kafka.TBKafkaProducerTemplate; -import org.thingsboard.server.kafka.TbKafkaSettings; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; +import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -44,20 +44,20 @@ import org.thingsboard.server.kafka.TbNodeIdProvider; public class KafkaTransportQueueProvider implements TransportQueueProvider { private final TbKafkaSettings kafkaSettings; - private final TbNodeIdProvider nodeIdProvider; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, - TbNodeIdProvider nodeIdProvider, + TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; - this.nodeIdProvider = nodeIdProvider; + this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -68,14 +68,14 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-transport-api-" + nodeIdProvider.getNodeId()); + requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("consumer-transport-api-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("transport-node-" + nodeIdProvider.getNodeId()); + responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder @@ -93,8 +93,8 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-rule-engine-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.clientId("transport-node-rule-engine-"+ serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); return requestBuilder.build(); } @@ -102,8 +102,8 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("producer-tb-core-" + nodeIdProvider.getNodeId()); - requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } @@ -111,9 +111,9 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { public TbQueueConsumer> getTransportNotificationsConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); - responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + nodeIdProvider.getNodeId()); - responseBuilder.clientId("consumer-transport-" + nodeIdProvider.getNodeId()); - responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); + responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); return responseBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java similarity index 88% rename from common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java index ba93bf96ef..9fdec01830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java @@ -13,13 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; -import org.thingsboard.server.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java index b53c3ac28b..c4d610cab1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java @@ -13,16 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. diff --git a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java similarity index 84% rename from common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java index de69e1819a..2aa18a62ca 100644 --- a/common/queue/src/main/java/org/thingsboard/server/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.provider; +package org.thingsboard.server.queue.provider; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index 5be2f76258..864e7b2ab9 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; * Created by ashvayka on 18.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Component public class CoapTransportContext extends TransportContext { diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index d725d47a5f..9323cfa7a0 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -36,7 +36,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')") @Slf4j public class CoapTransportService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index 9df658224a..c8e7f76450 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -61,7 +61,7 @@ import java.util.function.Consumer; * @author Andrew Shvayka */ @RestController -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j public class DeviceApiController { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index 3d344d8dc5..28c38bdda6 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -29,7 +29,7 @@ import javax.annotation.PostConstruct; * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')") @Component public class HttpTransportContext extends TransportContext { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 107639f302..b058a1c260 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -15,33 +15,23 @@ */ package org.thingsboard.server.transport.mqtt; -import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.ssl.SslHandler; -import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; -import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.kafka.TbNodeIdProvider; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - /** * Created by ashvayka on 04.10.18. */ @Slf4j -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @Component +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") public class MqttTransportContext extends TransportContext { @Getter diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index 8f752b5066..fb8a0b70be 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -36,7 +36,7 @@ import javax.annotation.PreDestroy; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')") @Slf4j public class MqttTransportService { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index ab67beee0b..c49b9c5bd3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,7 +20,9 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.kafka.TbNodeIdProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -32,15 +34,16 @@ import java.util.concurrent.Executors; */ @Slf4j @Data -public class TransportContext { +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'") +public abstract class TransportContext { protected final ObjectMapper mapper = new ObjectMapper(); @Autowired private TransportService transportService; - @Autowired - private TbNodeIdProvider nodeIdProvider; + private TbServiceInfoProvider serviceInfoProvider; @Getter private ExecutorService executor; @@ -58,7 +61,7 @@ public class TransportContext { } public String getNodeId() { - return nodeIdProvider.getNodeId(); + return serviceInfoProvider.getServiceId(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index f1b73f30af..fcb630d041 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -16,16 +16,16 @@ package org.thingsboard.server.common.transport.service; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.TbQueueCallback; -import org.thingsboard.server.TbQueueConsumer; -import org.thingsboard.server.TbQueueMsgMetadata; -import org.thingsboard.server.TbQueueProducer; -import org.thingsboard.server.TbQueueRequestTemplate; -import org.thingsboard.server.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,10 +34,10 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.discovery.PartitionService; -import org.thingsboard.server.discovery.ServiceType; -import org.thingsboard.server.discovery.TopicPartitionInfo; -import org.thingsboard.server.provider.TransportQueueProvider; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -46,7 +46,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToRuleEngineMsg; -import org.thingsboard.server.common.AsyncCallbackTemplate; +import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -66,6 +66,7 @@ import java.util.concurrent.TimeUnit; */ @Slf4j @Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'") public class DefaultTransportService implements TransportService { @Value("${transport.rate_limits.enabled}") diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java index 42a65440a0..fc1445539b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToRuleEngineMsgEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java index 31a32d8368..cc154f87b2 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/ToTransportMsgResponseDecoder.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java index 6c4b362f03..3971fdbf83 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiRequestEncoder.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.transport.service; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.kafka.TbKafkaEncoder; +import org.thingsboard.server.queue.kafka.TbKafkaEncoder; /** * Created by ashvayka on 05.10.18. diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java index 89d61fba56..5fdc58c068 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportApiResponseDecoder.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.common.transport.service; -import org.thingsboard.server.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.kafka.TbKafkaDecoder; +import org.thingsboard.server.queue.kafka.TbKafkaDecoder; import java.io.IOException; diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index 6dbca324e1..ec596db10d 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index 39dc0677e8..f44af2105e 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java index ce95ee6f1f..cd7225ad1f 100644 --- a/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java +++ b/transport/mqtt/src/main/java/org/thingsboard/server/mqtt/ThingsboardMqttTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.kafka"}) +@ComponentScan({"org.thingsboard.server.mqtt", "org.thingsboard.server.common", "org.thingsboard.server.transport.mqtt", "org.thingsboard.server.queue"}) public class ThingsboardMqttTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 721db2605d..f4a31449e7 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -62,11 +62,8 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -service: - type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport - queue: - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka or ? kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -74,18 +71,39 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - partitions: "${TB_QUEUE_CORE_PARTITIONS:100}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:100}" - notifications: - topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" + pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" + print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + transport: + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/ui/package-lock.json b/ui/package-lock.json index dccf637726..512bac811c 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1095,12 +1095,12 @@ "@flowjs/ng-flow": { "version": "2.7.8", "resolved": "https://registry.npmjs.org/@flowjs/ng-flow/-/ng-flow-2.7.8.tgz", - "integrity": "sha512-zO6jNvz41oMOJj9+1N+vLT0ytitbCtuGABJQRzQDOPXyRMmlSXfJ7om5oYOztyUFrr4jDpE4QFPt+r2/RFceCg==" + "integrity": "sha1-HZ+dH4Ks2lNgMowxW6z9YNv9mBk=" }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", - "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "integrity": "sha1-UkryQNGjYFJ7cwR17PoTRKpUDd4=", "dev": true, "requires": { "call-me-maybe": "^1.0.1", @@ -1424,7 +1424,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + "integrity": "sha1-+PLIh60Qv2f2NPAFtph/7TF5qsg=" }, "accepts": { "version": "1.3.7", @@ -1534,7 +1534,7 @@ "angular-carousel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/angular-carousel/-/angular-carousel-1.1.0.tgz", - "integrity": "sha512-UiLMgT7Ueqk4xpliF1gWt4dYKXezdJA1jyZPNsUWkOGO/dwLuKi284h3BgWl4CnaH7kEBw8L2gsBOyqbYaumNQ==" + "integrity": "sha1-PmlA5ovRio85L8Qx2XGSrDSIMdE=" }, "angular-cookies": { "version": "1.5.8", @@ -1555,7 +1555,7 @@ } }, "angular-fullscreen": { - "version": "git://github.com/fabiobiondi/angular-fullscreen.git#119b7fbac911d154fd56ace38ebe3432475e8a20", + "version": "git://github.com/fabiobiondi/angular-fullscreen.git#8217174565761d3566807bc60a73b5ca015b8cb6", "from": "git://github.com/fabiobiondi/angular-fullscreen.git#master" }, "angular-gridster": { @@ -1629,7 +1629,7 @@ "angular-translate": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.1.tgz", - "integrity": "sha512-Mw0kFBqsv5j8ItL9IhRZunIlVmIRW6iFsiTmRs9wGr2QTt8z4rehYlWyHos8qnXc/kyOYJiW50iH50CSNHGB9A==", + "integrity": "sha1-sp7Q0vm6xEB156rTKEFmxZ4VB5E=", "requires": { "angular": ">=1.2.26 <=1.7" } @@ -1637,7 +1637,7 @@ "angular-translate-handler-log": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-handler-log/-/angular-translate-handler-log-2.18.1.tgz", - "integrity": "sha512-TyKzCW4GubNazwCgLpCVXd2212CWdZOckf+aL5+gLuThPhVpOvlg18RSmz8MNPto3kwCcCw3LzShlZ6RX/MQRA==", + "integrity": "sha1-icu1mCeALYb4EVJ1+/iNbYiWsNQ=", "requires": { "angular-translate": "~2.18.1" } @@ -1645,7 +1645,7 @@ "angular-translate-interpolation-messageformat": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-interpolation-messageformat/-/angular-translate-interpolation-messageformat-2.18.1.tgz", - "integrity": "sha512-SlmyxLB/UUy7FWoGx5QJHrhq8fUu/xzCR0h/ngexOtXZopQjs1vm+TrFZ69d4c/LI7C91sfP4mq4ES29o1xCxA==", + "integrity": "sha1-FsUq4MYcJA8PJBZKBSGUPPi6QI4=", "requires": { "angular-translate": "~2.18.1", "messageformat": "~1.0.2" @@ -1654,7 +1654,7 @@ "angular-translate-loader-static-files": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-loader-static-files/-/angular-translate-loader-static-files-2.18.1.tgz", - "integrity": "sha512-5MuyzAROfc493kjLjKlLGLBzXiRmZIFbcWZGutDRxW5SRXSpwrH0u0hh0ENNnUyUQbe2vUspHNPIuZqlq8qIhw==", + "integrity": "sha1-rQw8iDsYsIm9uNsCu9Nm2QP4V8w=", "requires": { "angular-translate": "~2.18.1" } @@ -1662,7 +1662,7 @@ "angular-translate-storage-cookie": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-cookie/-/angular-translate-storage-cookie-2.18.1.tgz", - "integrity": "sha512-wiMaF/0OGN/3ilaYunfsqdLNpfGZEJK0fj4zT8yjD3XPq7Q9kM88xZ4XJiWKgodZShBljGCRzqgQbKMF7d1MLw==", + "integrity": "sha1-j8vaspb6gkkOALQorxp0ahf0QVY=", "requires": { "angular-cookies": ">=1.2.26 <1.8", "angular-translate": "~2.18.1" @@ -1671,7 +1671,7 @@ "angular-translate-storage-local": { "version": "2.18.1", "resolved": "https://registry.npmjs.org/angular-translate-storage-local/-/angular-translate-storage-local-2.18.1.tgz", - "integrity": "sha512-zPxcbIJ8tdWXtWNKLtaswynKid0w5le6WPMwiLWhgKPnyzOp/y5WLBW+JEfnZnkGE24yOGhJ6jVPgRNzelLgzg==", + "integrity": "sha1-lHQP5NgBq3gpopofBeHDkFTIcwM=", "requires": { "angular-translate": "~2.18.1", "angular-translate-storage-cookie": "~2.18.1" @@ -1750,7 +1750,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "integrity": "sha1-aALmJk79GMeQobDVF/DyYnvyyUo=", "dev": true }, "are-we-there-yet": { @@ -1766,7 +1766,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", "dev": true, "requires": { "sprintf-js": "~1.0.2" @@ -1781,7 +1781,7 @@ "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "integrity": "sha1-NgSLv/TntH4TZkQxbJlmnqWukfE=", "dev": true }, "arr-union": { @@ -1893,7 +1893,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1958,7 +1958,7 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "integrity": "sha1-bZUX654DDSQ2ZmZR6GvZ9vE1M8k=", "dev": true }, "attr-accept": { @@ -2174,7 +2174,7 @@ "base": { "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "integrity": "sha1-e95c7RRbbVUakNuH+DxVi060io8=", "dev": true, "requires": { "cache-base": "^1.0.1", @@ -2198,7 +2198,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2207,7 +2207,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -2216,7 +2216,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -2362,7 +2362,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2586,7 +2586,7 @@ "cache-base": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "integrity": "sha1-Cn9GQWgxyLZi7jb+TnxZ129marI=", "dev": true, "requires": { "collection-visit": "^1.0.0", @@ -2617,7 +2617,7 @@ "dependencies": { "callsites": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", "dev": true } @@ -2794,7 +2794,7 @@ "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "integrity": "sha1-+TNprouafOAv1B+q0MqDAzGQxGM=", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3121,7 +3121,7 @@ "concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3181,7 +3181,7 @@ "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=", "dev": true }, "convert-source-map": { @@ -3208,7 +3208,7 @@ "copy-concurrently": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", - "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "integrity": "sha1-kilzmMrjSTf8r9bsgTnBgFHwteA=", "dev": true, "requires": { "aproba": "^1.1.1", @@ -3427,7 +3427,7 @@ "create-react-class": { "version": "15.6.3", "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", - "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", + "integrity": "sha1-LXMjf7P5cK5uvgEanmb0bbyoADY=", "requires": { "fbjs": "^0.8.9", "loose-envify": "^1.3.1", @@ -3555,7 +3555,7 @@ "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", "dev": true, "requires": { "ms": "2.0.0" @@ -3633,7 +3633,7 @@ "define-property": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "integrity": "sha1-1Flono1lS6d+AqgX+HENcCyxbp0=", "dev": true, "requires": { "is-descriptor": "^1.0.2", @@ -3643,7 +3643,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3652,7 +3652,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -3661,7 +3661,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -3724,7 +3724,7 @@ "delegate": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", - "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + "integrity": "sha1-tmtxwxWFIuirV0T3INjKDCr1kWY=" }, "delegates": { "version": "1.0.0", @@ -3880,7 +3880,7 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "integrity": "sha1-PTH1AZGmdJ3RN1p/Ui6CPULlTto=", "dev": true }, "domelementtype": { @@ -4035,7 +4035,7 @@ "errno": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "integrity": "sha1-RoTXF3mtOa8Xfj8AeZb3xnyFJhg=", "dev": true, "requires": { "prr": "~1.0.1" @@ -4554,7 +4554,7 @@ "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "integrity": "sha1-AHo7n9vCs7uH5IeeoZyS/b05Qs8=", "dev": true, "requires": { "estraverse": "^4.1.0" @@ -4769,7 +4769,7 @@ "external-editor": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "integrity": "sha1-BFURz9jRM/OEZnPRBHwVTiFK09U=", "requires": { "chardet": "^0.4.0", "iconv-lite": "^0.4.17", @@ -5197,7 +5197,7 @@ "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", - "integrity": "sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==", + "integrity": "sha1-4y/AMKLM7kSmtTcTCNpUvgs5fSc=", "dev": true }, "fs-write-stream-atomic": { @@ -5238,8 +5238,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5260,14 +5259,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5282,20 +5279,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5412,8 +5406,7 @@ "inherits": { "version": "2.0.4", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5425,7 +5418,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5440,7 +5432,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5448,14 +5439,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.9.0", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5474,7 +5463,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5564,8 +5552,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5577,7 +5564,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5663,8 +5649,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5700,7 +5685,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5720,7 +5704,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5764,14 +5747,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -5790,7 +5771,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", "dev": true }, "functional-red-black-tree": { @@ -6738,7 +6719,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha1-7uJfVtscnsYIXgwid4CD9Zar+Sc=" }, "inline-style-prefixer": { "version": "2.0.5", @@ -6788,7 +6769,7 @@ "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "integrity": "sha1-YQ88ksk1nOHbYW5TgAjSP/NRWOY=", "dev": true, "requires": { "loose-envify": "^1.0.0" @@ -6890,7 +6871,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "integrity": "sha1-76ouqdqg16suoTqXsritUf776L4=", "dev": true }, "is-callable": { @@ -6934,7 +6915,7 @@ "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "integrity": "sha1-Nm2CQN3kh8pRgjsaufB6EKeCUco=", "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", @@ -6945,7 +6926,7 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "integrity": "sha1-cpyR4thXt6QZofmqZWhcTDP1hF0=", "dev": true } } @@ -7053,7 +7034,7 @@ "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "integrity": "sha1-LBY7P6+xtgbZ0Xko8FwqHDjgdnc=", "dev": true, "requires": { "isobject": "^3.0.1" @@ -7120,7 +7101,7 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "integrity": "sha1-0YUOuXkezRjmGCzhKjDzlmNLsZ0=", "dev": true }, "is-word-character": { @@ -7260,7 +7241,7 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "integrity": "sha1-u4Z8+zRQ5pEHwTHRxRS6s9yLyqk=", "dev": true }, "json-schema": { @@ -7277,7 +7258,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=", "dev": true }, "json-stable-stringify-without-jsonify": { @@ -7517,7 +7498,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8050,7 +8031,7 @@ "messageformat-parser": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-1.1.0.tgz", - "integrity": "sha512-Hwem6G3MsKDLS1FtBRGIs8T50P1Q00r3srS6QJePCFbad9fq0nYxwf3rnU2BreApRGhmpKMV7oZI06Sy1c9TPA==" + "integrity": "sha1-E7oiUKdrvejg/KDbs0dflcWUqQo=" }, "methods": { "version": "1.1.2", @@ -8092,7 +8073,7 @@ "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=", "dev": true }, "mime-db": { @@ -8113,7 +8094,7 @@ "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + "integrity": "sha1-ggyGo5M0ZA6ZUWkovQP8qIBX0CI=" }, "min-document": { "version": "2.19.0", @@ -8170,7 +8151,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", "requires": { "brace-expansion": "^1.1.7" } @@ -8265,7 +8246,7 @@ "is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "integrity": "sha1-p0cPnkJnM9gb2B4RVSZOOjUHyrQ=", "dev": true, "requires": { "is-plain-object": "^2.0.4" @@ -8343,7 +8324,7 @@ "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "integrity": "sha1-uHqKpPwN6P5r6IiVs4mD/yZb0Rk=", "dev": true, "requires": { "arr-diff": "^4.0.0", @@ -8495,7 +8476,7 @@ } }, "ngFlowchart": { - "version": "git://github.com/thingsboard/ngFlowchart.git#b941e4ed38c226019890b7b0802b71c2b147f0e0", + "version": "git://github.com/thingsboard/ngFlowchart.git#1343a7478961f68280d81f0ecda4e722a2068e0f", "from": "git://github.com/thingsboard/ngFlowchart.git#master" }, "ngclipboard": { @@ -8555,7 +8536,7 @@ "no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", + "integrity": "sha1-YLgTOWvjmz8SiKTB7V0efSi0ZKw=", "dev": true, "requires": { "lower-case": "^1.1.1" @@ -8574,7 +8555,7 @@ "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "integrity": "sha1-mA9vcthSEaU0fGsrwYxbhMPrR+8=", "requires": { "encoding": "^0.1.11", "is-stream": "^1.0.1" @@ -8775,7 +8756,7 @@ "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "integrity": "sha1-CKfyqL9zRgR3mp76StXMcXq7lUs=", "dev": true, "requires": { "are-we-there-yet": "~1.1.2", @@ -9092,7 +9073,7 @@ "osenv": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "integrity": "sha1-hc36+uso6Gd/QW4odZK18/SepBA=", "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -9520,7 +9501,7 @@ "postcss-loader": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", - "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", + "integrity": "sha1-a5eUPkfHLYRfqeA/Jzdz1OjdbC0=", "dev": true, "requires": { "loader-utils": "^1.1.0", @@ -9823,7 +9804,7 @@ "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", + "integrity": "sha1-I4Hts2ifelPWUxkAYPz4ItLzaP8=", "dev": true }, "process": { @@ -9846,7 +9827,7 @@ "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "integrity": "sha1-BktyYCsY+Q8pGSuLG8QY/9Hr078=", "requires": { "asap": "~2.0.3" } @@ -9926,7 +9907,7 @@ "pumpify": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", - "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "integrity": "sha1-NlE74karJ1cLGjdKXOJ4v9dDcM4=", "dev": true, "requires": { "duplexify": "^3.6.0", @@ -10091,7 +10072,7 @@ "rc-menu": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-5.1.4.tgz", - "integrity": "sha512-ZUkUNda70GtTXcQDiO3rSDdk3sgIwDwzPUm5dVM8nRH/j84qv0BVBkIUwIBu8+s+G3G9lWLurRqh22dCqZPeOA==", + "integrity": "sha1-5d8I/ouDPoFGkTX/E7MKuPIf88Y=", "requires": { "babel-runtime": "6.x", "classnames": "2.x", @@ -10122,7 +10103,7 @@ "rc-trigger": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-1.11.5.tgz", - "integrity": "sha512-MBuUPw1nFzA4K7jQOwb7uvFaZFjXGd00EofUYiZ+l/fgKVq8wnLC0lkv36kwqM7vfKyftRo2sh7cWVpdPuNnnw==", + "integrity": "sha1-+I+fhODnn44O8cjRv4rCIItxViA=", "requires": { "babel-runtime": "6.x", "create-react-class": "15.x", @@ -10281,7 +10262,7 @@ "react-transition-group": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-1.2.1.tgz", - "integrity": "sha512-CWaL3laCmgAFdxdKbhhps+c0HRGF4c+hdM4H23+FI1QBNUyx/AMeIJGWorehPNSaKnQNOAxL7PQmqMu78CDj3Q==", + "integrity": "sha1-4R9yslf5IbITIpp3TfRmEjRsfKY=", "requires": { "chain-function": "^1.0.0", "dom-helpers": "^3.2.0", @@ -10293,7 +10274,7 @@ "reactcss": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/reactcss/-/reactcss-1.2.3.tgz", - "integrity": "sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==", + "integrity": "sha1-wAATh15Vexzw39mjaKHD2rO1SN0=", "requires": { "lodash": "^4.0.1" } @@ -10476,7 +10457,7 @@ "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + "integrity": "sha1-vgWtf5v30i4Fb5cmzuUBf78Z4uk=" }, "regenerator-transform": { "version": "0.14.1", @@ -10490,7 +10471,7 @@ "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "integrity": "sha1-H07OJ+ALC2XgJHpoEOaoXYOldSw=", "dev": true, "requires": { "extend-shallow": "^3.0.2", @@ -10795,7 +10776,7 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "integrity": "sha1-uKSCXVvbH8P29Twrwz+BOIaBx7w=", "dev": true }, "retry": { @@ -10899,7 +10880,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "sass-graph": { "version": "2.2.4", @@ -11233,7 +11214,7 @@ "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "integrity": "sha1-ZJIufFZbDhQgS6GqfWlkJ40lGC0=", "dev": true, "requires": { "base": "^0.11.1", @@ -11269,7 +11250,7 @@ "snapdragon-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "integrity": "sha1-bBdfhv8UvbByRWPo88GwIaKGhTs=", "dev": true, "requires": { "define-property": "^1.0.0", @@ -11289,7 +11270,7 @@ "is-accessor-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "integrity": "sha1-FpwvbT3x+ZJhgHI2XJsOofaHhlY=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11298,7 +11279,7 @@ "is-data-descriptor": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "integrity": "sha1-2Eh2Mh0Oet0DmQQGq7u9NrqSaMc=", "dev": true, "requires": { "kind-of": "^6.0.0" @@ -11307,7 +11288,7 @@ "is-descriptor": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "integrity": "sha1-OxWXRqZmBLBPjIFSS6NlxfFNhuw=", "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", @@ -11320,7 +11301,7 @@ "snapdragon-util": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "integrity": "sha1-+VZHlIbyrNeXAGk/b3uAXkWrVuI=", "dev": true, "requires": { "kind-of": "^3.2.0" @@ -11340,7 +11321,7 @@ "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", - "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==", + "integrity": "sha1-2Xa76ACve9IK4IWY1YI5NQiZPA0=", "dev": true, "requires": { "faye-websocket": "^0.10.0", @@ -11464,7 +11445,7 @@ "spdx-expression-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=", "dev": true, "requires": { "spdx-exceptions": "^2.1.0", @@ -11558,7 +11539,7 @@ "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "integrity": "sha1-fLCd2jqGWFcFxks5pkZgOGguj+I=", "dev": true, "requires": { "extend-shallow": "^3.0.0" @@ -11698,7 +11679,7 @@ "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "integrity": "sha1-q5Pyeo3BPSjKyBXEYhQ6bZASrp4=", "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -11742,7 +11723,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "requires": { "safe-buffer": "~5.1.0" } @@ -12711,7 +12692,7 @@ "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", - "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==" + "integrity": "sha1-wiaIrtTqs83C3+rLtWFmBWCgCAQ=" }, "table": { "version": "5.4.6", @@ -12941,7 +12922,7 @@ "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "integrity": "sha1-bTQzWIl2jSGyvNoKonfO07G/rfk=", "requires": { "os-tmpdir": "~1.0.2" } @@ -12981,7 +12962,7 @@ "to-regex": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "integrity": "sha1-E8/dmzNlUvMLUfM6iuG0Knp1mc4=", "dev": true, "requires": { "define-property": "^2.0.2", @@ -13183,7 +13164,7 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", "dev": true } } @@ -13508,7 +13489,7 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", "dev": true, "requires": { "punycode": "^2.1.0" @@ -13570,7 +13551,7 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "integrity": "sha1-1QyMrHmhn7wg8pEfVuuXP04QBw8=", "dev": true }, "util": { @@ -14309,7 +14290,7 @@ "websocket-extensions": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", + "integrity": "sha1-XS/yKXcAPsaHpLhwc9+7rBRszyk=", "dev": true }, "whatwg-fetch": { @@ -14423,7 +14404,7 @@ "ws": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", + "integrity": "sha1-y9nm514J/F0skAFfIfDECHXg3VE=", "requires": { "options": ">=0.0.5", "ultron": "1.0.x" From e102b55bc139deff72c554f8e7f6e7c9f0c029a4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 Mar 2020 18:39:34 +0200 Subject: [PATCH 018/196] Thread-safety usage of Kafka Consumer --- .../state/DefaultDeviceStateService.java | 8 ++++--- .../queue/kafka/TBKafkaConsumerTemplate.java | 23 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 4e395f7090..5ee26bc294 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -293,12 +293,10 @@ public class DefaultDeviceStateService implements DeviceStateService { callback.onSuccess(); } } - } catch (Exception e) { log.trace("Failed to process queue msg: [{}]", proto, e); callback.onFailure(e); } - } @Override @@ -366,6 +364,10 @@ public class DefaultDeviceStateService implements DeviceStateService { } } } + log.info("Managing following partitions:"); + partitionedDevices.forEach((tpi, devices) -> { + log.info("[{}]: {} devices", tpi.getFullTopicName(), devices.size()); + }); } catch (Throwable t) { log.warn("Failed to init device states from DB", t); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index b4b1578939..09c29aa5e1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private volatile boolean subscribed; + private volatile Set partitions; @Getter private final String topic; @@ -74,29 +75,31 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { - createTopicIfNotExists(topic); - consumer.subscribe(Collections.singletonList(topic)); - subscribed = true; + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null)); + subscribed = false; } @Override public void subscribe(Set partitions) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(this::createTopicIfNotExists); - consumer.unsubscribe(); - consumer.subscribe(topicNames); - subscribed = true; + this.partitions = partitions; + subscribed = false; } @Override public List poll(long durationInMillis) { - if (!subscribed) { + if (!subscribed && partitions == null) { try { Thread.sleep(durationInMillis); } catch (InterruptedException e) { log.debug("Failed to await subscription", e); } } else { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(this::createTopicIfNotExists); + consumer.subscribe(topicNames); + subscribed = true; + } ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); if (records.count() > 0) { List result = new ArrayList<>(); From ce6ec889833db0802fa3b7af01b02ae95873c754 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 20 Mar 2020 18:00:57 +0200 Subject: [PATCH 019/196] Refactoring of the websocket and subscription services --- .../device/DeviceActorMessageProcessor.java | 3 +- .../server/actors/service/ActorService.java | 3 +- .../actors/service/DefaultActorService.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 84 ++- .../server/service/queue/TbMsgCallback.java | 13 + .../state/DefaultDeviceStateService.java | 15 +- .../DefaultLocalSubscriptionService.java | 230 ++++++ .../DefaultSubscriptionManagerService.java | 377 ++++++++++ .../LocalSubscriptionService.java | 36 + .../SubscriptionManagerService.java | 37 + .../subscription/TbAttributeSubscription.java | 50 ++ .../TbAttributeSubscriptionScope.java | 22 + .../service/subscription/TbSubscription.java | 52 ++ .../subscription/TbSubscriptionType.java | 20 + .../subscription/TbSubscriptionUtils.java | 247 +++++++ .../TbTimeseriesSubscription.java | 51 ++ .../DefaultTelemetrySubscriptionService.java | 657 ++---------------- .../DefaultTelemetryWebSocketService.java | 72 +- .../TelemetrySubscriptionService.java | 23 +- .../telemetry/sub/SubscriptionUpdate.java | 2 +- application/src/main/proto/cluster.proto | 62 -- .../src/main/resources/thingsboard.yml | 2 +- .../common/data/kv/BaseAttributeKvEntry.java | 4 + .../discovery/ClusterTopologyChangeEvent.java | 33 + .../ConsistentHashPartitionService.java | 119 +++- .../queue/discovery/PartitionService.java | 9 + .../queue/kafka/TBKafkaConsumerTemplate.java | 2 +- common/queue/src/main/proto/queue.proto | 97 ++- .../service/DefaultTransportService.java | 1 + .../api/RuleEngineTelemetryService.java | 2 - .../engine/telemetry/TbMsgAttributesNode.java | 3 - 31 files changed, 1541 insertions(+), 790 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java create mode 100644 application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 6b5dfbf6c3..11d556c819 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -224,7 +224,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - boolean reportDeviceActivity = false; + boolean reportDeviceActivity = true; TransportToDeviceActorMsg msg = wrapper.getMsg(); TbMsgCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { @@ -263,7 +263,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { callback.onSuccess(); } - //TODO 2.5 move this as a notification to the queue; private void reportLogicalDeviceActivity() { systemContext.getDeviceStateService().onDeviceActivity(deviceId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java index 890128e645..8adc6e8ac2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ActorService.java @@ -19,6 +19,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.SessionMsgProcessor; @@ -26,7 +27,7 @@ public interface ActorService extends SessionMsgProcessor { void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); - void onMsg(SendToClusterMsg msg); + void onMsg(TbActorMsg msg); void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index fc9eb537c1..cb681182f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; @@ -108,7 +109,7 @@ public class DefaultActorService implements ActorService { } @Override - public void onMsg(SendToClusterMsg msg) { + public void onMsg(TbActorMsg msg) { appActor.tell(msg, ActorRef.noSender()); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 07b7704d06..27d5453add 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -24,6 +24,7 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -34,6 +35,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.subscription.LocalSubscriptionService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -62,15 +67,19 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private final ActorSystemContext actorContext; private final DeviceStateService stateService; + private final LocalSubscriptionService localSubscriptionService; + private final SubscriptionManagerService subscriptionManagerService; private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService) { + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService, LocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService) { this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); this.actorContext = actorContext; this.stateService = stateService; + this.localSubscriptionService = localSubscriptionService; + this.subscriptionManagerService = subscriptionManagerService; } @PostConstruct @@ -108,8 +117,15 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } else if (toCoreMsg.hasDeviceStateServiceMsg()) { log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); + } else if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); } } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); callback.onFailure(e); } }); @@ -126,23 +142,10 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } } + log.info("Tb Core Consumer stopped."); }); } - private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(deviceStateServiceMsg); - } - stateService.onQueueMsg(deviceStateServiceMsg, callback); - } - - private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { - if (statsEnabled) { - stats.log(toDeviceActorMsg); - } - actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); - } - @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { @@ -161,4 +164,55 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } + private void forwardToLocalSubMgrService(TransportProtos.LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { + if (msg.hasSubUpdate()) { + localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToSubMgrService(TransportProtos.SubscriptionMgrMsgProto msg, TbMsgCallback callback) { + if (msg.hasAttributeSub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); + } else if (msg.hasTelemetrySub()) { + subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); + } else if (msg.hasSubClose()) { + TransportProtos.TbSubscriptionCloseProto closeProto = msg.getSubClose(); + subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); + } else if (msg.hasTsUpdate()) { + TransportProtos.TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + subscriptionManagerService.onTimeseriesDataUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); + } else if (msg.hasAttrUpdate()) { + TransportProtos.TbAttributeUpdateProto proto = msg.getAttrUpdate(); + subscriptionManagerService.onAttributesUpdate( + new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), + proto.getScope(), TbSubscriptionUtils.toAttributeKvList(proto.getDataList()), callback); + } else { + throwNotHandled(msg, callback); + } + } + + private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(deviceStateServiceMsg); + } + stateService.onQueueMsg(deviceStateServiceMsg, callback); + } + + private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg, TbMsgCallback callback) { + if (statsEnabled) { + stats.log(toDeviceActorMsg); + } + actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); + } + + private void throwNotHandled(Object msg, TbMsgCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java index 774377407a..85fec1d2ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java @@ -17,6 +17,19 @@ package org.thingsboard.server.service.queue; public interface TbMsgCallback { + TbMsgCallback EMPTY = new TbMsgCallback() { + + @Override + public void onSuccess() { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(); void onFailure(Throwable t); diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 5ee26bc294..51f544712e 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -92,7 +92,6 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; */ @Service @Slf4j -//TODO: refactor to use page links as cursor and not fetch all public class DefaultDeviceStateService implements DeviceStateService { private static final ObjectMapper json = new ObjectMapper(); @@ -150,7 +149,6 @@ public class DefaultDeviceStateService implements DeviceStateService { private volatile boolean clusterUpdatePending = false; private ListeningScheduledExecutorService queueExecutor; - private ConcurrentMap> tenantDevices = new ConcurrentHashMap<>(); private ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); @@ -378,7 +376,6 @@ public class DefaultDeviceStateService implements DeviceStateService { deviceStates.put(state.getDeviceId(), state); } - //TODO 2.5: review this method private void updateState() { long ts = System.currentTimeMillis(); Set deviceIds = new HashSet<>(deviceStates.keySet()); @@ -439,13 +436,9 @@ public class DefaultDeviceStateService implements DeviceStateService { deviceStates.remove(deviceId); deviceLastReportedActivity.remove(deviceId); deviceLastSavedActivity.remove(deviceId); - Set deviceIds = tenantDevices.get(tenantId); - if (deviceIds != null) { - deviceIds.remove(deviceId); - if (deviceIds.isEmpty()) { - tenantDevices.remove(tenantId); - } - } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + Set deviceIdSet = partitionedDevices.get(tpi); + deviceIdSet.remove(deviceId); } private ListenableFuture fetchDeviceState(Device device) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java new file mode 100644 index 0000000000..f9d60b37a5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java @@ -0,0 +1,230 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class DefaultLocalSubscriptionService implements LocalSubscriptionService { + + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); + + @Autowired + private TelemetryWebSocketService wsService; + + @Autowired + private EntityViewService entityViewService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbCoreQueueProvider coreQueueProvider; + + @Autowired + @Lazy + private SubscriptionManagerService subscriptionManagerService; + + private ExecutorService wsCallBackExecutor; + private TbQueueProducer> toCoreProducer; + + @PostConstruct + public void initExecutor() { + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + } + + @PreDestroy + public void shutdownExecutor() { + if (wsCallBackExecutor != null) { + wsCallBackExecutor.shutdownNow(); + } + } + + @Override + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + } + } + + @Override + @EventListener(ClusterTopologyChangeEvent.class) + public void onApplicationEvent(ClusterTopologyChangeEvent event) { + if (event.getServiceKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { + /* + * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. + * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. + * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically + * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. + * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache + * Since number of subscriptions is usually much less then number of devices that are pushing data. + */ + subscriptionsBySessionId.values().forEach(map -> map.values() + .forEach(sub -> pushSubscriptionToManagerService(sub, false))); + } + } + + //TODO 3.1: replace null callbacks with callbacks from websocket service. + @Override + public void addSubscription(TbSubscription subscription) { + EntityId entityId = subscription.getEntityId(); + // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges; + if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) { + subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription); + } + pushSubscriptionToManagerService(subscription, true); + registerSubscription(subscription); + } + + private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + if (pushToLocalService) { + subscriptionManagerService.addSubscription(subscription, TbMsgCallback.EMPTY); + } + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + } + } + + @Override + public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback) { + TbSubscription subscription = subscriptionsBySessionId + .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); + if (subscription != null) { + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription; + update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value)); + break; + case ATTRIBUTES: + TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription; + update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value)); + break; + } + wsService.sendWsMsg(sessionId, update); + } + callback.onSuccess(); + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsBySessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + if (sessionSubscriptions.isEmpty()) { + subscriptionsBySessionId.remove(sessionId); + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + // Subscription is managed on the same server; + subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbMsgCallback.EMPTY); + } else { + // Push to the queue; + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + } + + @Override + public void cancelAllSessionSubscriptions(String sessionId) { + Map subscriptions = subscriptionsBySessionId.get(sessionId); + if (subscriptions != null) { + Set toRemove = new HashSet<>(subscriptions.keySet()); + toRemove.forEach(id -> cancelSubscription(sessionId, id)); + } + } + + private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) { + EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId())); + + Map keyStates; + if (subscription.isAllKeys()) { + keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); + } else { + keyStates = subscription.getKeyStates().entrySet() + .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + return TbTimeseriesSubscription.builder() + .serviceId(subscription.getServiceId()) + .sessionId(subscription.getSessionId()) + .subscriptionId(subscription.getSubscriptionId()) + .tenantId(subscription.getTenantId()) + .entityId(entityView.getEntityId()) + .startTime(entityView.getStartTimeMs()) + .endTime(entityView.getEndTimeMs()) + .allKeys(false) + .keyStates(keyStates).build(); + } + + private void registerSubscription(TbSubscription subscription) { + Map sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()); + sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java new file mode 100644 index 0000000000..267ea14ac1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -0,0 +1,377 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.state.DefaultDeviceStateService; +import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.function.Predicate; + +@Slf4j +@Service +public class DefaultSubscriptionManagerService implements SubscriptionManagerService { + + @Autowired + private AttributesService attrService; + + @Autowired + private TimeseriesService tsService; + + @Autowired + private PartitionService partitionService; + + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + + @Autowired + private TbCoreQueueProvider coreQueueProvider; + + @Autowired + private LocalSubscriptionService localSubscriptionService; + + @Autowired + private DeviceStateService deviceStateService; + + @Autowired + private ActorService actorService; + + private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); + private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedSubscriptions = new ConcurrentHashMap<>(); + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); + + private ExecutorService tsCallBackExecutor; + private String serviceId; + private TbQueueProducer> toCoreProducer; + + @PostConstruct + public void initExecutor() { + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); + serviceId = serviceInfoProvider.getServiceId(); + toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + } + + @PreDestroy + public void shutdownExecutor() { + if (tsCallBackExecutor != null) { + tsCallBackExecutor.shutdownNow(); + } + } + + @Override + public void addSubscription(TbSubscription subscription, TbMsgCallback callback) { + log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", + subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); + if (currentPartitions.contains(tpi)) { + partitionedSubscriptions.computeIfAbsent(tpi, k -> ConcurrentHashMap.newKeySet()).add(subscription); + callback.onSuccess(); + } else { + log.warn("[{}][{}] Entity belongs to external partition. Probably rebalancing is in progress. Topic: {}" + , subscription.getTenantId(), subscription.getEntityId(), tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Entity belongs to external partition " + tpi.getFullTopicName() + "!")); + } + boolean newSubscription = subscriptionsByEntityId + .computeIfAbsent(subscription.getEntityId(), k -> ConcurrentHashMap.newKeySet()).add(subscription); + subscriptionsByWsSessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>()).put(subscription.getSubscriptionId(), subscription); + if (newSubscription) { + switch (subscription.getType()) { + case TIMESERIES: + handleNewTelemetrySubscription((TbTimeseriesSubscription) subscription); + break; + case ATTRIBUTES: + handleNewAttributeSubscription((TbAttributeSubscription) subscription); + break; + } + } + } + + @Override + public void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback) { + log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); + Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); + if (sessionSubscriptions != null) { + TbSubscription subscription = sessionSubscriptions.remove(subscriptionId); + if (subscription != null) { + removeSubscriptionFromEntityMap(subscription); + removeSubscriptionFromPartitionMap(subscription); + if (sessionSubscriptions.isEmpty()) { + subscriptionsByWsSessionId.remove(sessionId); + } + } else { + log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); + } + } else { + log.debug("[{}] No session subscriptions found!", sessionId); + } + callback.onSuccess(); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + Set removedPartitions = new HashSet<>(currentPartitions); + removedPartitions.removeAll(partitionChangeEvent.getPartitions()); + + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set subs = partitionedSubscriptions.remove(partition); + if (subs != null) { + subs.forEach(this::removeSubscriptionFromEntityMap); + } + }); + } + + @Override + public void onTimeseriesDataUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { + return (TbTimeseriesSubscription) s; + } else { + return null; + } + }, s -> true, s -> { + List subscriptionUpdate = null; + for (TsKvEntry kv : ts) { + if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(kv); + } + } + return subscriptionUpdate; + }); + callback.onSuccess(); + } + + @Override + public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback) { + onLocalSubUpdate(entityId, + s -> { + if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { + return (TbAttributeSubscription) s; + } else { + return null; + } + }, + s -> (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope().name())), + s -> { + List subscriptionUpdate = null; + for (AttributeKvEntry kv : attributes) { + if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { + if (subscriptionUpdate == null) { + subscriptionUpdate = new ArrayList<>(); + } + subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); + } + } + return subscriptionUpdate; + }); + if (entityId.getEntityType() == EntityType.DEVICE) { + if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { + for (AttributeKvEntry attribute : attributes) { + if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { + deviceStateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); + } + } + } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { + DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); + actorService.onMsg(notificationMsg); + } + } + callback.onSuccess(); + } + + private void onLocalSubUpdate(EntityId entityId, + Function castFunction, + Predicate filterFunction, + Function> processFunction) { + Set entitySubscriptions = subscriptionsByEntityId.get(entityId); + if (entitySubscriptions != null) { + entitySubscriptions.stream().map(castFunction).filter(Objects::nonNull).filter(filterFunction).forEach(s -> { + List subscriptionUpdate = processFunction.apply(s); + if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { + if (serviceId.equals(s.getServiceId())) { + SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); + localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbMsgCallback.EMPTY); + } else { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); + toCoreProducer.send(tpi, toProto(s, subscriptionUpdate), null); + } + } + }); + } else { + log.debug("[{}] No device subscriptions to process!", entityId); + } + } + + private boolean isInTimeRange(TbTimeseriesSubscription subscription, long kvTime) { + return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) + && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); + } + + private void removeSubscriptionFromEntityMap(TbSubscription sub) { + Set entitySubSet = subscriptionsByEntityId.get(sub.getEntityId()); + if (entitySubSet != null) { + entitySubSet.remove(sub); + if (entitySubSet.isEmpty()) { + subscriptionsByEntityId.remove(sub.getEntityId()); + } + } + } + + private void removeSubscriptionFromPartitionMap(TbSubscription sub) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, sub.getTenantId(), sub.getEntityId()); + Set subs = partitionedSubscriptions.get(tpi); + if (subs != null) { + subs.remove(sub); + } + } + + private void handleNewAttributeSubscription(TbAttributeSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote attribute subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + final Map keyStates = subscription.getKeyStates(); + DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + List missedUpdates = new ArrayList<>(); + values.forEach(latestEntry -> { + if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { + missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); + } + }); + if (!missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); + } + + private void handleNewTelemetrySubscription(TbTimeseriesSubscription subscription) { + log.trace("[{}][{}][{}] Processing remote telemetry subscription for entity [{}]", + serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); + + long curTs = System.currentTimeMillis(); + List queries = new ArrayList<>(); + subscription.getKeyStates().forEach((key, value) -> { + if (curTs > value) { + long startTs = subscription.getStartTime() > 0 ? Math.max(subscription.getStartTime(), value + 1L) : (value + 1L); + long endTs = subscription.getEndTime() > 0 ? Math.min(subscription.getEndTime(), curTs) : curTs; + queries.add(new BaseReadTsKvQuery(key, startTs, endTs, 0, 1000, Aggregation.NONE)); + } + }); + if (!queries.isEmpty()) { + DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries), + missedUpdates -> { + if (missedUpdates != null && !missedUpdates.isEmpty()) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); + toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + } + }, + e -> log.error("Failed to fetch missed updates.", e), + tsCallBackExecutor); + } + } + + private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { + TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); + + builder.setSessionId(subscription.getSessionId()); + builder.setSubscriptionId(subscription.getSubscriptionId()); + + Map> data = new TreeMap<>(); + for (TsKvEntry tsEntry : updates) { + List values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>()); + Object[] value = new Object[2]; + value[0] = tsEntry.getTs(); + value[1] = tsEntry.getValueAsString(); + values.add(value); + } + + data.forEach((key, value) -> { + TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder(); + dataBuilder.setKey(key); + value.forEach(v -> { + Object[] array = (Object[]) v; + dataBuilder.addTs((long) array[0]); + dataBuilder.addValue((String) array[1]); + }); + builder.addData(dataBuilder.build()); + }); + + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToLocalSubscriptionServiceMsg( + LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) + .build(); + return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java new file mode 100644 index 0000000000..a80fb0db92 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +public interface LocalSubscriptionService { + + void addSubscription(TbSubscription subscription); + + void cancelSubscription(String sessionId, int subscriptionId); + + void cancelAllSessionSubscriptions(String sessionId); + + void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback); + + void onApplicationEvent(PartitionChangeEvent event); + + void onApplicationEvent(ClusterTopologyChangeEvent event); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java new file mode 100644 index 0000000000..bb236141f8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.queue.TbMsgCallback; + +import java.util.List; + +public interface SubscriptionManagerService extends ApplicationListener { + + void addSubscription(TbSubscription subscription, TbMsgCallback callback); + + void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback); + + void onTimeseriesDataUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback); + + void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback); +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java new file mode 100644 index 0000000000..83a86efefc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscription.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbAttributeSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final TbAttributeSubscriptionScope scope; + + @Builder + public TbAttributeSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, TbAttributeSubscriptionScope scope) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ATTRIBUTES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.scope = scope; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java new file mode 100644 index 0000000000..d23cc3581d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +public enum TbAttributeSubscriptionScope { + + CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE + +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java new file mode 100644 index 0000000000..22b37ff690 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscription.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +@Data +@AllArgsConstructor +public abstract class TbSubscription { + + private final String serviceId; + private final String sessionId; + private final int subscriptionId; + private final TenantId tenantId; + private final EntityId entityId; + private final TbSubscriptionType type; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbSubscription that = (TbSubscription) o; + return subscriptionId == that.subscriptionId && + sessionId.equals(that.sessionId) && + tenantId.equals(that.tenantId) && + entityId.equals(that.entityId) && + type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(sessionId, subscriptionId, tenantId, entityId, type); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java new file mode 100644 index 0000000000..d7a4715460 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +public enum TbSubscriptionType { + TIMESERIES, ATTRIBUTES +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java new file mode 100644 index 0000000000..d0b4d713f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java @@ -0,0 +1,247 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionKetStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesSubscriptionProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; +import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +public class TbSubscriptionUtils { + + public static ToCoreMsg toNewSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionProto subscriptionProto = TbSubscriptionProto.newBuilder() + .setServiceId(subscription.getServiceId()) + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()) + .setTenantIdMSB(subscription.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(subscription.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(subscription.getEntityId().getEntityType().name()) + .setEntityIdMSB(subscription.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(subscription.getEntityId().getId().getLeastSignificantBits()).build(); + + switch (subscription.getType()) { + case TIMESERIES: + TbTimeseriesSubscription tSub = (TbTimeseriesSubscription) subscription; + TbTimeSeriesSubscriptionProto.Builder tSubProto = TbTimeSeriesSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(tSub.isAllKeys()); + tSub.getKeyStates().forEach((key, value) -> tSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + tSubProto.setStartTime(tSub.getStartTime()); + tSubProto.setEndTime(tSub.getEndTime()); + msgBuilder.setTelemetrySub(tSubProto.build()); + break; + case ATTRIBUTES: + TbAttributeSubscription aSub = (TbAttributeSubscription) subscription; + TbAttributeSubscriptionProto.Builder aSubProto = TbAttributeSubscriptionProto.newBuilder() + .setSub(subscriptionProto) + .setAllKeys(aSub.isAllKeys()) + .setScope(aSub.getScope().name()); + aSub.getKeyStates().forEach((key, value) -> aSubProto.addKeyStates( + TbSubscriptionKetStateProto.newBuilder().setKey(key).setTs(value).build())); + msgBuilder.setAttributeSub(aSubProto.build()); + break; + } + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toCloseSubscriptionProto(TbSubscription subscription) { + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + TbSubscriptionCloseProto closeProto = TbSubscriptionCloseProto.newBuilder() + .setSessionId(subscription.getSessionId()) + .setSubscriptionId(subscription.getSubscriptionId()).build(); + msgBuilder.setSubClose(closeProto); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static TbSubscription fromProto(TbAttributeSubscriptionProto attributeSub) { + TbSubscriptionProto subProto = attributeSub.getSub(); + TbAttributeSubscription.TbAttributeSubscriptionBuilder builder = TbAttributeSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.scope(TbAttributeSubscriptionScope.valueOf(attributeSub.getScope())); + builder.allKeys(attributeSub.getAllKeys()); + Map keyStates = new HashMap<>(); + attributeSub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.keyStates(keyStates); + return builder.build(); + } + + public static TbSubscription fromProto(TbTimeSeriesSubscriptionProto telemetrySub) { + TbSubscriptionProto subProto = telemetrySub.getSub(); + TbTimeseriesSubscription.TbTimeseriesSubscriptionBuilder builder = TbTimeseriesSubscription.builder() + .serviceId(subProto.getServiceId()) + .sessionId(subProto.getSessionId()) + .subscriptionId(subProto.getSubscriptionId()) + .entityId(EntityIdFactory.getByTypeAndUuid(subProto.getEntityType(), new UUID(subProto.getEntityIdMSB(), subProto.getEntityIdLSB()))) + .tenantId(new TenantId(new UUID(subProto.getTenantIdMSB(), subProto.getTenantIdLSB()))); + + builder.allKeys(telemetrySub.getAllKeys()); + Map keyStates = new HashMap<>(); + telemetrySub.getKeyStatesList().forEach(ksProto -> keyStates.put(ksProto.getKey(), ksProto.getTs())); + builder.startTime(telemetrySub.getStartTime()); + builder.endTime(telemetrySub.getEndTime()); + builder.keyStates(keyStates); + return builder.build(); + } + + public static SubscriptionUpdate fromProto(TbSubscriptionUpdateProto proto) { + if (proto.getErrorCode() > 0) { + return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); + } else { + Map> data = new TreeMap<>(); + proto.getDataList().forEach(v -> { + List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); + for (int i = 0; i < v.getTsCount(); i++) { + Object[] value = new Object[2]; + value[0] = v.getTs(i); + value[1] = v.getValue(i); + values.add(value); + } + }); + return new SubscriptionUpdate(proto.getSubscriptionId(), data); + } + } + + public static ToCoreMsg toTimeseriesUpdateProto(TenantId tenantId, EntityId entityId, List ts) { + TbTimeSeriesUpdateProto.Builder builder = TbTimeSeriesUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setTsUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + public static ToCoreMsg toAttributesUpdateProto(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TbAttributeUpdateProto.Builder builder = TbAttributeUpdateProto.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setScope(scope); + attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); + + SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); + msgBuilder.setAttrUpdate(builder); + return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); + } + + private static TsKvProto.Builder toKeyValueProto(long ts, KvEntry attr) { + KeyValueProto.Builder dataBuilder = KeyValueProto.newBuilder(); + dataBuilder.setKey(attr.getKey()); + dataBuilder.setType(KeyValueType.forNumber(attr.getDataType().ordinal())); + switch (attr.getDataType()) { + case BOOLEAN: + attr.getBooleanValue().ifPresent(dataBuilder::setBoolV); + break; + case LONG: + attr.getLongValue().ifPresent(dataBuilder::setLongV); + break; + case DOUBLE: + attr.getDoubleValue().ifPresent(dataBuilder::setDoubleV); + break; + case JSON: + attr.getJsonValue().ifPresent(dataBuilder::setJsonV); + break; + case STRING: + attr.getStrValue().ifPresent(dataBuilder::setStringV); + break; + } + return TsKvProto.newBuilder().setTs(ts).setKv(dataBuilder); + } + + public static EntityId toEntityId(String entityType, long entityIdMSB, long entityIdLSB) { + return EntityIdFactory.getByTypeAndUuid(entityType, new UUID(entityIdMSB, entityIdLSB)); + } + + public static List toTsKvEntityList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BasicTsKvEntry(proto.getTs(), getKvEntry(proto.getKv())))); + return result; + } + + public static List toAttributeKvList(List dataList) { + List result = new ArrayList<>(dataList.size()); + dataList.forEach(proto -> result.add(new BaseAttributeKvEntry(getKvEntry(proto.getKv()), proto.getTs()))); + return result; + } + + private static KvEntry getKvEntry(KeyValueProto proto) { + KvEntry entry = null; + DataType type = DataType.values()[proto.getType().getNumber()]; + switch (type) { + case BOOLEAN: + entry = new BooleanDataEntry(proto.getKey(), proto.getBoolV()); + break; + case LONG: + entry = new LongDataEntry(proto.getKey(), proto.getLongV()); + break; + case DOUBLE: + entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleV()); + break; + case STRING: + entry = new StringDataEntry(proto.getKey(), proto.getStringV()); + break; + case JSON: + entry = new JsonDataEntry(proto.getKey(), proto.getJsonV()); + break; + } + return entry; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java new file mode 100644 index 0000000000..0be63f7b65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbTimeseriesSubscription.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import lombok.Builder; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +public class TbTimeseriesSubscription extends TbSubscription { + + @Getter private final boolean allKeys; + @Getter private final Map keyStates; + @Getter private final long startTime; + @Getter private final long endTime; + + @Builder + public TbTimeseriesSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, + boolean allKeys, Map keyStates, long startTime, long endTime) { + super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.TIMESERIES); + this.allKeys = allKeys; + this.keyStates = keyStates; + this.startTime = startTime; + this.endTime = endTime; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 04952a3bd1..d94eb3f2b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -18,71 +18,44 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.telemetry.sub.Subscription; -import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; -import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -91,8 +64,7 @@ import java.util.stream.Collectors; @Slf4j public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptionService { - @Autowired - private TelemetryWebSocketService wsService; + private final Set currentPartitions = ConcurrentHashMap.newKeySet(); @Autowired private AttributesService attrService; @@ -101,23 +73,24 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private TimeseriesService tsService; @Autowired - private EntityViewService entityViewService; + private TbCoreQueueProvider coreQueueProvider; @Autowired - @Lazy - private DeviceStateService stateService; + private PartitionService partitionService; @Autowired - @Lazy - private ActorService actorService; - + private SubscriptionManagerService subscriptionManagerService; + + private TbQueueProducer> toCoreProducer; + private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; @PostConstruct public void initExecutor() { - tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); - wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); + tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); + wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); + toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); } @PreDestroy @@ -130,65 +103,12 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } - private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); - private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); - - @Override - public void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub) { - long startTime = 0L; - long endTime = 0L; - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TelemetryFeature.TIMESERIES.equals(sub.getType())) { - EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(entityId.getId())); - entityId = entityView.getEntityId(); - startTime = entityView.getStartTimeMs(); - endTime = entityView.getEndTimeMs(); - sub = getUpdatedSubscriptionState(entityId, sub, entityView); - } - //TODO 2.5 - Optional server = Optional.empty();//routingService.resolveById(entityId); - Subscription subscription; - if (server.isPresent()) { - ServerAddress address = server.get(); - log.trace("[{}] Forwarding subscription [{}] for [{}] entity [{}] to [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId, address); - subscription = new Subscription(sub, true, address, startTime, endTime); - tellNewSubscription(address, sessionId, subscription); - } else { - log.trace("[{}] Registering local subscription [{}] for [{}] entity [{}]", sessionId, sub.getSubscriptionId(), entityId.getEntityType().name(), entityId); - subscription = new Subscription(sub, true, null, startTime, endTime); - } - registerSubscription(sessionId, entityId, subscription); - } - - private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) { - Map keyStates; - if (sub.isAllKeys()) { - keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L)); - } else { - keyStates = sub.getKeyStates().entrySet() - .stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - return new SubscriptionState(sub.getWsSessionId(), sub.getSubscriptionId(), sub.getTenantId(), entityId, sub.getType(), false, keyStates, sub.getScope()); - } - @Override - public void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId) { - cleanupLocalWsSessionSubscriptions(sessionId); - } - - @Override - public void removeSubscription(String sessionId, int subscriptionId) { - log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - Subscription subscription = sessionSubscriptions.remove(subscriptionId); - if (subscription != null) { - processSubscriptionRemoval(sessionId, sessionSubscriptions, subscription); - } else { - log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); - } - } else { - log.debug("[{}] No session subscriptions found!", sessionId); + @EventListener(PartitionChangeEvent.class) + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); } } @@ -201,14 +121,14 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onTimeseriesUpdate(entityId, ts)); + addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addMainCallback(saveFuture, callback); - addWsCallback(saveFuture, success -> onAttributesUpdate(entityId, scope, attributes)); + addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes)); } @Override @@ -235,355 +155,23 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio , System.currentTimeMillis())), callback); } - @Override - public void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes) { - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - deviceId, DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); - } - - @Override - public void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionProto proto; - try { - proto = ClusterAPIProtos.SubscriptionProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - Map statesMap = proto.getKeyStatesList().stream().collect( - Collectors.toMap(ClusterAPIProtos.SubscriptionKetStateProto::getKey, ClusterAPIProtos.SubscriptionKetStateProto::getTs)); - Subscription subscription = new Subscription( - new SubscriptionState(proto.getSessionId(), proto.getSubscriptionId(), - new TenantId(UUID.fromString(proto.getTenantId())), - EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - TelemetryFeature.valueOf(proto.getType()), proto.getAllKeys(), statesMap, proto.getScope()), - false, new ServerAddress(serverAddress.getHost(), serverAddress.getPort(), serverAddress.getServerType())); - - addRemoteWsSubscription(serverAddress, proto.getSessionId(), subscription); - } - - @Override - public void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionUpdateProto proto; - try { - proto = ClusterAPIProtos.SubscriptionUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - SubscriptionUpdate update = convert(proto); - String sessionId = proto.getSessionId(); - log.trace("[{}] Processing remote subscription onUpdate [{}]", sessionId, update); - Optional subOpt = getSubscription(sessionId, update.getSubscriptionId()); - if (subOpt.isPresent()) { - updateSubscriptionState(sessionId, subOpt.get(), update); - wsService.sendWsMsg(sessionId, update); - } - } - - @Override - public void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SubscriptionCloseProto proto; - try { - proto = ClusterAPIProtos.SubscriptionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - removeSubscription(proto.getSessionId(), proto.getSubscriptionId()); - } - - @Override - public void onRemoteSessionClose(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.SessionCloseProto proto; - try { - proto = ClusterAPIProtos.SessionCloseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - cleanupRemoteWsSessionSubscriptions(proto.getSessionId()); - } - - @Override - public void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.AttributeUpdateProto proto; - try { - proto = ClusterAPIProtos.AttributeUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onAttributesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), proto.getScope(), - proto.getDataList().stream().map(this::toAttribute).collect(Collectors.toList())); - } - - @Override - public void onRemoteTsUpdate(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.TimeseriesUpdateProto proto; - try { - proto = ClusterAPIProtos.TimeseriesUpdateProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - onTimeseriesUpdate(EntityIdFactory.getByTypeAndId(proto.getEntityType(), proto.getEntityId()), - proto.getDataList().stream().map(this::toTimeseries).collect(Collectors.toList())); - } - - @Override - public void onClusterUpdate() { - log.trace("Processing cluster onUpdate msg!"); - Iterator>> deviceIterator = subscriptionsByEntityId.entrySet().iterator(); - while (deviceIterator.hasNext()) { - Map.Entry> e = deviceIterator.next(); - Set subscriptions = e.getValue(); - //TODO 2.5 - Optional newAddressOptional = Optional.empty();// routingService.resolveById(e.getKey()); - if (newAddressOptional.isPresent()) { - newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions)); - } else { - checkSubscriptionsPrevAddress(subscriptions); - } - if (subscriptions.size() == 0) { - log.trace("[{}] No more subscriptions for this device on current server.", e.getKey()); - deviceIterator.remove(); - } - } - } - - private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set subscriptions) { - Iterator subscriptionIterator = subscriptions.iterator(); - while (subscriptionIterator.hasNext()) { - Subscription s = subscriptionIterator.next(); - if (s.isLocal()) { - if (!newAddress.equals(s.getServer())) { - log.trace("[{}] Local subscription is now handled on new server [{}]", s.getWsSessionId(), newAddress); - s.setServer(newAddress); - tellNewSubscription(newAddress, s.getWsSessionId(), s); - } - } else { - log.trace("[{}] Remote subscription is now handled on new server address: [{}]", s.getWsSessionId(), newAddress); - subscriptionIterator.remove(); - //TODO: onUpdate state of subscription by WsSessionId and other maps. - } - } - } - - private void checkSubscriptionsPrevAddress(Set subscriptions) { - for (Subscription s : subscriptions) { - if (s.isLocal() && s.getServer() != null) { - log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer()); - s.setServer(null); - } else { - log.trace("[{}] Remote subscription is on up to date server address.", s.getWsSessionId()); - } - } - } - - private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address); - registerSubscription(sessionId, entityId, subscription); - if (subscription.getType() == TelemetryFeature.ATTRIBUTES) { - final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getSub().getTenantId(), entityId, DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { - List missedUpdates = new ArrayList<>(); - values.forEach(latestEntry -> { - if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { - missedUpdates.add(new BasicTsKvEntry(latestEntry.getLastUpdateTs(), latestEntry)); - } - }); - if (!missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); - } else if (subscription.getType() == TelemetryFeature.TIMESERIES) { - long curTs = System.currentTimeMillis(); - List queries = new ArrayList<>(); - subscription.getKeyStates().entrySet().forEach(e -> { - if (curTs > e.getValue()) { - queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE)); - } else { - log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs); - } - }); - if (!queries.isEmpty()) { - DonAsynchron.withCallback(tsService.findAll(subscription.getSub().getTenantId(), entityId, queries), - missedUpdates -> { - if (missedUpdates != null && !missedUpdates.isEmpty()) { - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates)); - } - }, - e -> log.error("Failed to fetch missed updates.", e), - tsCallBackExecutor); - } - } - } - - private void onAttributesUpdate(EntityId entityId, String scope, List attributes) { - //TODO 2.5 - Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalAttributesUpdate(entityId, scope, attributes); - if (entityId.getEntityType() == EntityType.DEVICE && DataConstants.SERVER_SCOPE.equalsIgnoreCase(scope)) { - for (AttributeKvEntry attribute : attributes) { - if (attribute.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - stateService.onDeviceInactivityTimeoutUpdate(new DeviceId(entityId.getId()), attribute.getLongValue().orElse(0L)); - } - } - } + private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); } else { - tellRemoteAttributesUpdate(serverAddress.get(), entityId, scope, attributes); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); } } - private void onTimeseriesUpdate(EntityId entityId, List ts) { - //TODO 2.5 - Optional serverAddress = Optional.empty();//routingService.resolveById(entityId); - if (!serverAddress.isPresent()) { - onLocalTimeseriesUpdate(entityId, ts); + private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + if (currentPartitions.contains(tpi)) { + subscriptionManagerService.onTimeseriesDataUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); } else { - tellRemoteTimeseriesUpdate(serverAddress.get(), entityId, ts); - } - } - - private void onLocalAttributesUpdate(EntityId entityId, String scope, List attributes) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.ATTRIBUTES == s.getType() && (StringUtils.isEmpty(s.getScope()) || scope.equals(s.getScope())), s -> { - List subscriptionUpdate = null; - for (AttributeKvEntry kv : attributes) { - if (s.isAllKeys() || s.getKeyStates().containsKey(kv.getKey())) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(new BasicTsKvEntry(kv.getLastUpdateTs(), kv)); - } - } - return subscriptionUpdate; - }); - } - - private void onLocalTimeseriesUpdate(EntityId entityId, List ts) { - onLocalSubUpdate(entityId, s -> TelemetryFeature.TIMESERIES == s.getType(), s -> { - List subscriptionUpdate = null; - for (TsKvEntry kv : ts) { - if (isInTimeRange(s, kv.getTs()) && (s.isAllKeys() || s.getKeyStates().containsKey((kv.getKey())))) { - if (subscriptionUpdate == null) { - subscriptionUpdate = new ArrayList<>(); - } - subscriptionUpdate.add(kv); - } - } - return subscriptionUpdate; - }); - } - - private boolean isInTimeRange(Subscription subscription, long kvTime) { - return (subscription.getStartTime() == 0 || subscription.getStartTime() <= kvTime) - && (subscription.getEndTime() == 0 || subscription.getEndTime() >= kvTime); - } - - private void onLocalSubUpdate(EntityId entityId, Predicate filter, Function> f) { - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - deviceSubscriptions.stream().filter(filter).forEach(s -> { - String sessionId = s.getWsSessionId(); - List subscriptionUpdate = f.apply(s); - if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { - SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - if (s.isLocal()) { - updateSubscriptionState(sessionId, s, update); - wsService.sendWsMsg(sessionId, update); - } else { - tellRemoteSubUpdate(s.getServer(), sessionId, update); - } - } - }); - } else { - log.debug("[{}] No device subscriptions to process!", entityId); - } - } - - private void updateSubscriptionState(String sessionId, Subscription subState, SubscriptionUpdate update) { - log.trace("[{}] updating subscription state {} using onUpdate {}", sessionId, subState, update); - update.getLatestValues().entrySet().forEach(e -> subState.setKeyState(e.getKey(), e.getValue())); - } - - private void registerSubscription(String sessionId, EntityId entityId, Subscription subscription) { - Set deviceSubscriptions = subscriptionsByEntityId.computeIfAbsent(entityId, k -> ConcurrentHashMap.newKeySet()); - deviceSubscriptions.add(subscription); - Map sessionSubscriptions = subscriptionsByWsSessionId.computeIfAbsent(sessionId, k -> new ConcurrentHashMap<>()); - sessionSubscriptions.put(subscription.getSubscriptionId(), subscription); - } - - private void cleanupLocalWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, true); - } - - private void cleanupRemoteWsSessionSubscriptions(String sessionId) { - cleanupWsSessionSubscriptions(sessionId, false); - } - - private void cleanupWsSessionSubscriptions(String sessionId, boolean localSession) { - log.debug("[{}] Removing all subscriptions for particular session.", sessionId); - Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); - if (sessionSubscriptions != null) { - int sessionSubscriptionSize = sessionSubscriptions.size(); - - for (Subscription subscription : sessionSubscriptions.values()) { - EntityId entityId = subscription.getEntityId(); - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - deviceSubscriptions.remove(subscription); - if (deviceSubscriptions.isEmpty()) { - subscriptionsByEntityId.remove(entityId); - } - } - subscriptionsByWsSessionId.remove(sessionId); - log.debug("[{}] Removed {} subscriptions for particular session.", sessionId, sessionSubscriptionSize); - - if (localSession) { - notifyWsSubscriptionClosed(sessionId, sessionSubscriptions); - } - } else { - log.debug("[{}] No subscriptions found!", sessionId); - } - } - - private void notifyWsSubscriptionClosed(String sessionId, Map sessionSubscriptions) { - Set affectedServers = new HashSet<>(); - for (Subscription subscription : sessionSubscriptions.values()) { - if (subscription.getServer() != null) { - affectedServers.add(subscription.getServer()); - } - } - for (ServerAddress address : affectedServers) { - log.debug("[{}] Going to onSubscriptionUpdate [{}] server about session close event", sessionId, address); - tellRemoteSessionClose(address, sessionId); - } - } - - private void processSubscriptionRemoval(String sessionId, Map sessionSubscriptions, Subscription subscription) { - EntityId entityId = subscription.getEntityId(); - if (subscription.isLocal() && subscription.getServer() != null) { - tellRemoteSubClose(subscription.getServer(), sessionId, subscription.getSubscriptionId()); - } - if (sessionSubscriptions.isEmpty()) { - log.debug("[{}] Removed last subscription for particular session.", sessionId); - subscriptionsByWsSessionId.remove(sessionId); - } else { - log.debug("[{}] Removed session subscription.", sessionId); - } - Set deviceSubscriptions = subscriptionsByEntityId.get(entityId); - if (deviceSubscriptions != null) { - boolean result = deviceSubscriptions.remove(subscription); - if (result) { - if (deviceSubscriptions.size() == 0) { - log.debug("[{}] Removed last subscription for particular device.", sessionId); - subscriptionsByEntityId.remove(entityId); - } else { - log.debug("[{}] Removed device subscription.", sessionId); - } - } else { - log.debug("[{}] Subscription not found!", sessionId); - } - } else { - log.debug("[{}] No device subscriptions found!", sessionId); + TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); } } @@ -613,167 +201,4 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } }, wsCallBackExecutor); } - - private void tellNewSubscription(ServerAddress address, String sessionId, Subscription sub) { - ClusterAPIProtos.SubscriptionProto.Builder builder = ClusterAPIProtos.SubscriptionProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(sub.getSubscriptionId()); - builder.setTenantId(sub.getSub().getTenantId().getId().toString()); - builder.setEntityType(sub.getEntityId().getEntityType().name()); - builder.setEntityId(sub.getEntityId().getId().toString()); - builder.setType(sub.getType().name()); - builder.setAllKeys(sub.isAllKeys()); - if (sub.getScope() != null) { - builder.setScope(sub.getScope()); - } - sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates( - ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build())); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSubUpdate(ServerAddress address, String sessionId, SubscriptionUpdate update) { - ClusterAPIProtos.SubscriptionUpdateProto.Builder builder = ClusterAPIProtos.SubscriptionUpdateProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setSubscriptionId(update.getSubscriptionId()); - builder.setErrorCode(update.getErrorCode()); - if (update.getErrorMsg() != null) { - builder.setErrorMsg(update.getErrorMsg()); - } - update.getData().entrySet().forEach( - e -> { - ClusterAPIProtos.SubscriptionUpdateValueListProto.Builder dataBuilder = ClusterAPIProtos.SubscriptionUpdateValueListProto.newBuilder(); - - dataBuilder.setKey(e.getKey()); - e.getValue().forEach(v -> { - Object[] array = (Object[]) v; - dataBuilder.addTs((long) array[0]); - dataBuilder.addValue((String) array[1]); - }); - - builder.addData(dataBuilder.build()); - } - ); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteAttributesUpdate(ServerAddress address, EntityId entityId, String scope, List attributes) { - ClusterAPIProtos.AttributeUpdateProto.Builder builder = ClusterAPIProtos.AttributeUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setScope(scope); - attributes.forEach(v -> builder.addData(toKeyValueProto(v.getLastUpdateTs(), v).build())); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteTimeseriesUpdate(ServerAddress address, EntityId entityId, List ts) { - ClusterAPIProtos.TimeseriesUpdateProto.Builder builder = ClusterAPIProtos.TimeseriesUpdateProto.newBuilder(); - builder.setEntityId(entityId.getId().toString()); - builder.setEntityType(entityId.getEntityType().name()); - ts.forEach(v -> builder.addData(toKeyValueProto(v.getTs(), v).build())); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE, builder.build().toByteArray()); - } - - private void tellRemoteSessionClose(ServerAddress address, String sessionId) { - ClusterAPIProtos.SessionCloseProto proto = ClusterAPIProtos.SessionCloseProto.newBuilder().setSessionId(sessionId).build(); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private void tellRemoteSubClose(ServerAddress address, String sessionId, int subscriptionId) { - ClusterAPIProtos.SubscriptionCloseProto proto = ClusterAPIProtos.SubscriptionCloseProto.newBuilder().setSessionId(sessionId).setSubscriptionId(subscriptionId).build(); - //TODO 2.5 -// rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE, proto.toByteArray()); - } - - private ClusterAPIProtos.KeyValueProto.Builder toKeyValueProto(long ts, KvEntry attr) { - ClusterAPIProtos.KeyValueProto.Builder dataBuilder = ClusterAPIProtos.KeyValueProto.newBuilder(); - dataBuilder.setKey(attr.getKey()); - dataBuilder.setTs(ts); - dataBuilder.setValueType(attr.getDataType().ordinal()); - switch (attr.getDataType()) { - case BOOLEAN: - Optional booleanValue = attr.getBooleanValue(); - booleanValue.ifPresent(dataBuilder::setBoolValue); - break; - case LONG: - Optional longValue = attr.getLongValue(); - longValue.ifPresent(dataBuilder::setLongValue); - break; - case DOUBLE: - Optional doubleValue = attr.getDoubleValue(); - doubleValue.ifPresent(dataBuilder::setDoubleValue); - break; - case JSON: - Optional jsonValue = attr.getJsonValue(); - jsonValue.ifPresent(dataBuilder::setJsonValue); - break; - case STRING: - Optional stringValue = attr.getStrValue(); - stringValue.ifPresent(dataBuilder::setStrValue); - break; - } - return dataBuilder; - } - - private AttributeKvEntry toAttribute(ClusterAPIProtos.KeyValueProto proto) { - return new BaseAttributeKvEntry(getKvEntry(proto), proto.getTs()); - } - - private TsKvEntry toTimeseries(ClusterAPIProtos.KeyValueProto proto) { - return new BasicTsKvEntry(proto.getTs(), getKvEntry(proto)); - } - - private KvEntry getKvEntry(ClusterAPIProtos.KeyValueProto proto) { - KvEntry entry = null; - DataType type = DataType.values()[proto.getValueType()]; - switch (type) { - case BOOLEAN: - entry = new BooleanDataEntry(proto.getKey(), proto.getBoolValue()); - break; - case LONG: - entry = new LongDataEntry(proto.getKey(), proto.getLongValue()); - break; - case DOUBLE: - entry = new DoubleDataEntry(proto.getKey(), proto.getDoubleValue()); - break; - case STRING: - entry = new StringDataEntry(proto.getKey(), proto.getStrValue()); - break; - case JSON: - entry = new JsonDataEntry(proto.getKey(), proto.getJsonValue()); - break; - } - return entry; - } - - private SubscriptionUpdate convert(ClusterAPIProtos.SubscriptionUpdateProto proto) { - if (proto.getErrorCode() > 0) { - return new SubscriptionUpdate(proto.getSubscriptionId(), SubscriptionErrorCode.forCode(proto.getErrorCode()), proto.getErrorMsg()); - } else { - Map> data = new TreeMap<>(); - proto.getDataList().forEach(v -> { - List values = data.computeIfAbsent(v.getKey(), k -> new ArrayList<>()); - for (int i = 0; i < v.getTsCount(); i++) { - Object[] value = new Object[2]; - value[0] = v.getTs(i); - value[1] = v.getValue(i); - values.add(value); - } - }); - return new SubscriptionUpdate(proto.getSubscriptionId(), data); - } - } - - private Optional getSubscription(String sessionId, int subscriptionId) { - Subscription state = null; - Map subMap = subscriptionsByWsSessionId.get(sessionId); - if (subMap != null) { - state = subMap.get(subscriptionId); - } - return Optional.ofNullable(state); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index c4fe716ad9..51556602f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -42,24 +42,25 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.subscription.LocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; +import org.thingsboard.server.service.subscription.TbAttributeSubscription; +import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd; import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd; import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmd; import org.thingsboard.server.service.telemetry.cmd.TelemetryPluginCmdsWrapper; import org.thingsboard.server.service.telemetry.cmd.TimeseriesSubscriptionCmd; -import org.thingsboard.server.service.telemetry.exception.AccessDeniedException; -import org.thingsboard.server.service.telemetry.exception.EntityNotFoundException; -import org.thingsboard.server.service.telemetry.exception.InternalErrorException; import org.thingsboard.server.service.telemetry.exception.UnauthorizedException; import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; import javax.annotation.Nullable; @@ -70,7 +71,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -98,7 +98,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private TelemetrySubscriptionService subscriptionManager; + private LocalSubscriptionService subService; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; @@ -112,6 +112,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Autowired private TimeseriesService tsService; + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + @Value("${server.ws.limits.max_subscriptions_per_tenant:0}") private int maxSubscriptionsPerTenant; @Value("${server.ws.limits.max_subscriptions_per_customer:0}") @@ -127,9 +130,11 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; + private String serviceId; @PostConstruct public void initExecutor() { + serviceId = serviceInfoProvider.getServiceId(); executor = Executors.newWorkStealingPool(50); } @@ -153,7 +158,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi break; case CLOSED: wsSessionsMap.remove(sessionId); - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); processSessionClose(sessionRef); break; } @@ -334,8 +339,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState) + .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + subService.addSubscription(sub); } @Override @@ -421,8 +434,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.ATTRIBUTES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + TbAttributeSubscription sub = TbAttributeSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState) + .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + subService.addSubscription(sub); } @Override @@ -494,8 +515,16 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, true, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(true) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -520,12 +549,19 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { sendWsMsg(sessionRef, new SubscriptionUpdate(cmd.getCmdId(), data)); - Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); - SubscriptionState sub = new SubscriptionState(sessionId, cmd.getCmdId(), sessionRef.getSecurityCtx().getTenantId(), entityId, TelemetryFeature.TIMESERIES, false, subState, cmd.getScope()); - subscriptionManager.addLocalWsSubscription(sessionId, entityId, sub); + + TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionId) + .subscriptionId(cmd.getCmdId()) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .allKeys(false) + .keyStates(subState).build(); + subService.addSubscription(sub); } @Override @@ -544,9 +580,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void unsubscribe(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd, String sessionId) { if (cmd.getEntityId() == null || cmd.getEntityId().isEmpty()) { - subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId); + subService.cancelAllSessionSubscriptions(sessionId); } else { - subscriptionManager.removeSubscription(sessionId, cmd.getCmdId()); + subService.cancelSubscription(sessionId, cmd.getCmdId()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index d77a0e50dc..e905559516 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -15,34 +15,17 @@ */ package org.thingsboard.server.service.telemetry; +import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.telemetry.sub.SubscriptionState; /** * Created by ashvayka on 27.03.18. */ -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService { +public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener { - void addLocalWsSubscription(String sessionId, EntityId entityId, SubscriptionState sub); - - void cleanupLocalWsSessionSubscriptions(TelemetryWebSocketSessionRef sessionRef, String sessionId); - - void removeSubscription(String sessionId, int cmdId); - - void onNewRemoteSubscription(ServerAddress serverAddress, byte[] data); - - void onRemoteSubscriptionUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSubscriptionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteSessionClose(ServerAddress serverAddress, byte[] bytes); - - void onRemoteAttributesUpdate(ServerAddress serverAddress, byte[] bytes); - - void onRemoteTsUpdate(ServerAddress serverAddress, byte[] bytes); - - void onClusterUpdate(); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java index 68aaca34c2..992dc26af9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/SubscriptionUpdate.java @@ -75,7 +75,7 @@ public class SubscriptionUpdate { if (data == null) { return Collections.emptyMap(); } else { - return data.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { + return data.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { List data = e.getValue(); Object[] latest = (Object[]) data.get(data.size() - 1); return (long) latest[0]; diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto index 46526e8e1c..f0ca4d6d7c 100644 --- a/application/src/main/proto/cluster.proto +++ b/application/src/main/proto/cluster.proto @@ -64,69 +64,7 @@ enum MessageType { } // Messages related to CLUSTER_TELEMETRY_MESSAGE -message SubscriptionProto { - string sessionId = 1; - int32 subscriptionId = 2; - string entityType = 3; - string tenantId = 4; - string entityId = 5; - string type = 6; - bool allKeys = 7; - repeated SubscriptionKetStateProto keyStates = 8; - string scope = 9; -} - -message SubscriptionUpdateProto { - string sessionId = 1; - int32 subscriptionId = 2; - int32 errorCode = 3; - string errorMsg = 4; - repeated SubscriptionUpdateValueListProto data = 5; -} - -message AttributeUpdateProto { - string entityType = 1; - string entityId = 2; - string scope = 3; - repeated KeyValueProto data = 4; -} - -message TimeseriesUpdateProto { - string entityType = 1; - string entityId = 2; - repeated KeyValueProto data = 4; -} - -message SessionCloseProto { - string sessionId = 1; -} - -message SubscriptionCloseProto { - string sessionId = 1; - int32 subscriptionId = 2; -} - -message SubscriptionKetStateProto { - string key = 1; - int64 ts = 2; -} - -message SubscriptionUpdateValueListProto { - string key = 1; - repeated int64 ts = 2; - repeated string value = 3; -} -message KeyValueProto { - string key = 1; - int64 ts = 2; - int32 valueType = 3; - string strValue = 4; - int64 longValue = 5; - double doubleValue = 6; - bool boolValue = 7; - string jsonValue = 8; -} message FromDeviceRPCResponseProto { int64 requestIdMSB = 1; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fc49716d25..8a4d735d26 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -412,7 +412,7 @@ audit-log: state: defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" - persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:true}" js: evaluator: "${JS_EVALUATOR:local}" # local/remote diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 5639f98d01..2993d81154 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -32,6 +32,10 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { this.lastUpdateTs = lastUpdateTs; } + public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { + this(kv, lastUpdateTs); + } + @Override public long getLastUpdateTs() { return lastUpdateTs; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java new file mode 100644 index 0000000000..fb379c213a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.Set; + + +public class ClusterTopologyChangeEvent extends ApplicationEvent { + + @Getter + private final Set serviceKeys; + + public ClusterTopologyChangeEvent(Object source, Set serviceKeys) { + super(source); + this.serviceKeys = serviceKeys; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 97b660a48c..f5efec30d4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -66,6 +66,10 @@ public class ConsistentHashPartitionService implements PartitionService { //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + private Map tbCoreNotificationTopics = new HashMap<>(); + private Map tbRuleEngineNotificationTopics = new HashMap<>(); + private List currentOtherServices; + private HashFunction hashFunction; public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, ApplicationEventPublisher applicationEventPublisher) { @@ -85,12 +89,11 @@ public class ConsistentHashPartitionService implements PartitionService { @Override public List getCurrentPartitions(ServiceType serviceType) { ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); - TenantId tenantId = getTenantId(currentService); + TenantId tenantId = getSystemOrIsolatedTenantId(currentService); ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); List partitions = myPartitions.get(serviceKey); List topicPartitions = new ArrayList<>(); for (Integer partition : partitions) { - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); @@ -112,34 +115,16 @@ public class ConsistentHashPartitionService implements PartitionService { return buildTopicPartitionInfo(serviceType, tenantId, partition); } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { - return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); - } - - private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { - boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceType)); - tpi.partition(partition); - if (isolated) { - tpi.tenantId(tenantId); - } - return tpi.build(); - } - @Override public void recalculatePartitions(ServiceInfo currentService, List otherServices) { logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); - Map> newCircles = new HashMap<>(ServiceType.values().length); - for (ServiceType serverType : ServiceType.values()) { - newCircles.put(serverType, new ConsistentHashCircle<>()); - } - addNode(newCircles, currentService); + Map> circles = new HashMap<>(); + addNode(circles, currentService); for (ServiceInfo other : otherServices) { - addNode(newCircles, other); - TenantId tenantId = getTenantId(other); + TenantId tenantId = getSystemOrIsolatedTenantId(other); + addNode(circles, other); if (!tenantId.isNullUid()) { isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); for (String serviceType : other.getServiceTypesList()) { @@ -149,12 +134,14 @@ public class ConsistentHashPartitionService implements PartitionService { } } ConcurrentMap> oldPartitions = myPartitions; + TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); myPartitions = new ConcurrentHashMap<>(); partitionSizes.forEach((type, size) -> { + ServiceKey myServiceKey = new ServiceKey(type, myTenantId); for (int i = 0; i < size; i++) { - ServiceInfo serviceInfo = resolveByPartitionIdx(newCircles.get(type), i); + ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceKey), i); if (currentService.equals(serviceInfo)) { - ServiceKey serviceKey = new ServiceKey(type, getTenantId(serviceInfo)); + ServiceKey serviceKey = new ServiceKey(type, getSystemOrIsolatedTenantId(serviceInfo)); myPartitions.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(i); } } @@ -165,13 +152,81 @@ public class ConsistentHashPartitionService implements PartitionService { Set tpiList = partitions.stream() .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) .collect(Collectors.toSet()); + // Adding notifications topic for every @TopicPartitionInfo list + tpiList.add(getNotificationsTopic(serviceKey.getServiceType(), serviceInfoProvider.getServiceId())); applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } + }); + + if (currentOtherServices == null) { + currentOtherServices = new ArrayList<>(otherServices); + } else { + Set changes = new HashSet<>(); + Map> currentMap = getServiceKeyListMap(currentOtherServices); + Map> newMap = getServiceKeyListMap(otherServices); + currentOtherServices = otherServices; + currentMap.forEach((key, list) -> { + if (!list.equals(newMap.get(key))) { + changes.add(key); + + } + }); + currentMap.keySet().forEach(newMap::remove); + changes.addAll(newMap.keySet()); + if (!changes.isEmpty()) { + applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + } + } + } + + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); + services.forEach(serviceInfo -> { + for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + } + }); + return currentMap; + } + + @Override + public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { + switch (serviceType) { + case TB_CORE: + return tbCoreNotificationTopics.computeIfAbsent(serviceId, + id -> buildTopicPartitionInfo(serviceType, serviceId)); + case TB_RULE_ENGINE: + return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, + id -> buildTopicPartitionInfo(serviceType, serviceId)); + default: + return buildTopicPartitionInfo(serviceType, serviceId); + } + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, String serviceId) { + return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { + return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); + } + + private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { + boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); + tpi.topic(partitionTopics.get(serviceType)); + tpi.partition(partition); + if (isolated) { + tpi.tenantId(tenantId); + } + return tpi.build(); } private void logServiceInfo(TransportProtos.ServiceInfo server) { - TenantId tenantId = getTenantId(server); + TenantId tenantId = getSystemOrIsolatedTenantId(server); if (tenantId.isNullUid()) { log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList()); } else { @@ -179,21 +234,23 @@ public class ConsistentHashPartitionService implements PartitionService { } } - private TenantId getTenantId(TransportProtos.ServiceInfo serviceInfo) { + private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) { return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); } - private void addNode(Map> circles, ServiceInfo instance) { + private void addNode(Map> circles, ServiceInfo instance) { + TenantId tenantId = getSystemOrIsolatedTenantId(instance); for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); for (int i = 0; i < virtualNodesSize; i++) { - circles.get(serviceType).put(hash(instance, i).asLong(), instance); + circles.computeIfAbsent(serviceKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); } } } private ServiceInfo resolveByPartitionIdx(ConsistentHashCircle circle, Integer partitionIdx) { - if (circle.isEmpty()) { + if (circle == null || circle.isEmpty()) { return null; } Long hash = hashFunction.newHasher().putInt(partitionIdx).hash().asLong(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index f8613f5fb4..46864af79f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -36,4 +36,13 @@ public interface PartitionService { * @param otherServices - all other discovered services {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} */ void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); + + /** + * Each Service should start a consumer for messages that target individual service instance based on serviceId. + * This topic is likely to have single partition, and is always assigned to the service. + * @param tbCore + * @param serviceId + * @return + */ + TopicPartitionInfo getNotificationsTopic(ServiceType tbCore, String serviceId); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index 09c29aa5e1..52086b3058 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 17f8b9c31c..19e8d64e87 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -239,6 +239,79 @@ message DeviceActorToTransportMsg { ToServerRpcResponseMsg toServerResponse = 7; } +/** + * TB Core Data Structures + */ + +message TbSubscriptionProto { + string serviceId = 1; + string sessionId = 2; + int32 subscriptionId = 3; + string entityType = 4; + int64 tenantIdMSB = 5; + int64 tenantIdLSB = 6; + int64 entityIdMSB = 7; + int64 entityIdLSB = 8; +} + +message TbTimeSeriesSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + int64 startTime = 4; + int64 endTime = 5; +} + +message TbAttributeSubscriptionProto { + TbSubscriptionProto sub = 1; + bool allKeys = 2; + repeated TbSubscriptionKetStateProto keyStates = 3; + string scope = 4; +} + +message TbSubscriptionUpdateProto { + string sessionId = 1; + int32 subscriptionId = 2; + int32 errorCode = 3; + string errorMsg = 4; + repeated TbSubscriptionUpdateValueListProto data = 5; +} + +message TbAttributeUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + string scope = 6; + repeated TsKvProto data = 7; +} + +message TbTimeSeriesUpdateProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + int64 tenantIdMSB = 4; + int64 tenantIdLSB = 5; + repeated TsKvProto data = 6; +} + +message TbSubscriptionCloseProto { + string sessionId = 1; + int32 subscriptionId = 2; +} + +message TbSubscriptionKetStateProto { + string key = 1; + int64 ts = 2; +} + +message TbSubscriptionUpdateValueListProto { + string key = 1; + repeated int64 ts = 2; + repeated string value = 3; +} + /** * TB Core to TB Core messages */ @@ -253,27 +326,41 @@ message DeviceStateServiceMsgProto { bool deleted = 7; } +message SubscriptionMgrMsgProto { + TbTimeSeriesSubscriptionProto telemetrySub = 1; + TbAttributeSubscriptionProto attributeSub = 2; + TbSubscriptionCloseProto subClose = 3; + TbTimeSeriesUpdateProto tsUpdate = 4; + TbAttributeUpdateProto attrUpdate = 5; +} + +message LocalSubscriptionServiceMsgProto { + TbSubscriptionUpdateProto subUpdate = 1; +} + /** * Main messages; */ /* Request from Transport Service to ThingsBoard Core Service */ message TransportApiRequestMsg { - ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; - ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; - GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; + ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; + ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; + GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; } /* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { - ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; - GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; + ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; + GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; } /* Messages that are handled by ThingsBoard Core Service */ message ToCoreMsg { TransportToDeviceActorMsg toDeviceActorMsg = 1; DeviceStateServiceMsgProto deviceStateServiceMsg = 2; + SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; + LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 4; } /* Messages that are handled by ThingsBoard RuleEngine Service */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index fcb630d041..d8d827aa52 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -136,6 +136,7 @@ public class DefaultTransportService implements TransportService { log.warn("Failed to process the notification.", e); } }); + transportNotificationsConsumer.commit(); } catch (Exception e) { log.warn("Failed to obtain messages from queue.", e); try { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index f57849fa35..d2e19652a2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -44,6 +44,4 @@ public interface RuleEngineTelemetryService { void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); - void onSharedAttributesUpdate(TenantId tenantId, DeviceId deviceId, Set attributes); - } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 46a68d3e26..4ddd082f3e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -65,9 +65,6 @@ public class TbMsgAttributesNode implements TbNode { String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); - if (msg.getOriginator().getEntityType() == EntityType.DEVICE && DataConstants.SHARED_SCOPE.equals(config.getScope())) { - ctx.getTelemetryService().onSharedAttributesUpdate(ctx.getTenantId(), new DeviceId(msg.getOriginator().getId()), attributes); - } } @Override From 99d322d4497f9d37d3fd1ff27578c6edeb948913 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Mar 2020 10:35:54 +0200 Subject: [PATCH 020/196] Refactoring of the rule engine --- .../server/actors/app/AppActor.java | 18 +---- .../device/DeviceActorMessageProcessor.java | 24 +----- .../actors/ruleChain/DefaultTbContext.java | 26 ++++--- .../actors/ruleChain/RuleChainActor.java | 7 +- .../RuleChainActorMessageProcessor.java | 13 ++-- .../actors/shared/EntityActorsManager.java | 6 ++ .../server/actors/tenant/TenantActor.java | 35 +++++---- .../server/controller/BaseController.java | 6 +- .../controller/RuleChainController.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 24 +++--- .../server/service/queue/MsgPackCallback.java | 1 + .../service/rpc/DefaultDeviceRpcService.java | 6 +- .../script/RuleNodeJsScriptEngine.java | 2 +- .../state/DefaultDeviceStateService.java | 8 +- .../service/state/DeviceStateService.java | 2 +- .../DefaultLocalSubscriptionService.java | 2 +- .../DefaultSubscriptionManagerService.java | 3 +- .../LocalSubscriptionService.java | 2 +- .../SubscriptionManagerService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../BaseRuleChainTransactionService.java | 2 +- .../msg/TransportToDeviceActorMsgWrapper.java | 2 +- ...AbstractRuleEngineFlowIntegrationTest.java | 21 ++--- ...actRuleEngineLifecycleIntegrationTest.java | 10 +-- .../script/RuleNodeJsScriptEngineTest.java | 15 ++-- .../server/common/msg/MsgType.java | 8 +- .../thingsboard/server/common/msg/TbMsg.java | 49 ++++-------- .../QueueToRuleEngineMsg.java} | 6 +- .../common/msg}/queue/TbMsgCallback.java | 2 +- common/queue/src/main/proto/queue.proto | 5 +- .../service/DefaultTransportService.java | 77 ++++++++++++++++--- .../common/transport/util}/JsonUtils.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 3 +- .../engine/rest/TbRedisQueueProcessor.java | 3 +- .../TbSynchronizationBeginNode.java | 3 +- .../rule/engine/action/TbAlarmNodeTest.java | 12 +-- .../engine/filter/TbJsFilterNodeTest.java | 7 +- .../engine/filter/TbJsSwitchNodeTest.java | 3 +- .../engine/mail/TbMsgToEmailNodeTest.java | 3 +- .../TbGetCustomerAttributeNodeTest.java | 17 ++-- .../transform/TbChangeOriginatorNodeTest.java | 7 +- .../transform/TbTransformMsgNodeTest.java | 7 +- 43 files changed, 249 insertions(+), 209 deletions(-) rename common/message/src/main/java/org/thingsboard/server/common/msg/{system/ServiceToRuleEngineMsg.java => queue/QueueToRuleEngineMsg.java} (85%) rename {application/src/main/java/org/thingsboard/server/service => common/message/src/main/java/org/thingsboard/server/common/msg}/queue/TbMsgCallback.java (94%) rename {application/src/main/java/org/thingsboard/server/utils => common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util}/JsonUtils.java (97%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 06f936a9e6..9d80d63b2a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -20,14 +20,9 @@ import akka.actor.LocalActorRef; import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; -import akka.actor.SupervisorStrategy.Directive; import akka.actor.Terminated; -import akka.event.Logging; -import akka.event.LoggingAdapter; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; @@ -43,17 +38,12 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - public class AppActor extends RuleChainManagerActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); @@ -97,8 +87,8 @@ public class AppActor extends RuleChainManagerActor { case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); break; - case SERVICE_TO_RULE_ENGINE_MSG: - onServiceToRuleEngineMsg((ServiceToRuleEngineMsg) msg); + case QUEUE_TO_RULE_ENGINE_MSG: + onQueueToRuleEngineMsg((QueueToRuleEngineMsg) msg); break; case TRANSPORT_TO_DEVICE_ACTOR_MSG: case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: @@ -145,7 +135,7 @@ public class AppActor extends RuleChainManagerActor { // } } - private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) { + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { // this may be a notification about system entities created. // log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 11d556c819..1844539b2c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -65,12 +65,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.utils.JsonUtils; +import org.thingsboard.server.common.transport.util.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -224,6 +224,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { + //TODO 2.5 boolean reportDeviceActivity = true; TransportToDeviceActorMsg msg = wrapper.getMsg(); TbMsgCallback callback = wrapper.getCallback(); @@ -329,23 +330,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handlePostAttributesRequest(ActorContext context, SessionInfoProto sessionInfo, PostAttributeMsg postAttributes) { - JsonObject json = JsonUtils.getJsonObject(postAttributes.getKvList()); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, defaultMetaData.copy(), - TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - - private void handlePostTelemetryRequest(ActorContext context, SessionInfoProto sessionInfo, PostTelemetryMsg postTelemetry) { - for (TsKvListProto tsKv : postTelemetry.getTsKvListList()) { - JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsgMetaData metaData = defaultMetaData.copy(); - metaData.putValue("ts", tsKv.getTs() + ""); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); - pushToRuleEngine(context, tbMsg); - } - } - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { UUID sessionId = getSessionId(sessionInfo); JsonObject json = new JsonObject(); @@ -354,7 +338,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { TbMsgMetaData requestMetaData = defaultMetaData.copy(); requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, 0L); + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 2de491c89d..337a234e2d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.util.StringUtils; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; @@ -47,12 +46,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.cluster.ServerType; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -71,7 +69,6 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; import java.util.Collections; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -146,23 +143,25 @@ class DefaultTbContext implements TbContext { @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), 0); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(),null); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), 0); + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), + data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), null); } @Override public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new ServiceToRuleEngineMsg(getTenantId(), msg))); + mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new QueueToRuleEngineMsg(getTenantId(), msg))); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), + getActionMetaData(ruleNodeId), TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process customer created msg: " + e); } @@ -171,7 +170,8 @@ class DefaultTbContext implements TbContext { public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), + TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process device created msg: " + e); } @@ -180,7 +180,8 @@ class DefaultTbContext implements TbContext { public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), + TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process asset created msg: " + e); } @@ -189,7 +190,8 @@ class DefaultTbContext implements TbContext { public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { try { ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), mapper.writeValueAsString(entityNode), null, null, 0L); + return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), + TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process alarm created msg: " + e); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 72faaef533..76f831be98 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.actors.ruleChain; -import akka.actor.ActorInitializationException; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; @@ -26,7 +25,7 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { @@ -43,8 +42,8 @@ public class RuleChainActor extends ComponentActor> msgs = consumer.poll(pollDuration); - if(msgs.isEmpty()){ + if (msgs.isEmpty()) { continue; } ConcurrentMap> ackMap = msgs.stream().collect( @@ -96,9 +101,10 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); try { TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); log.trace("Forwarding message to rule engine {}", toRuleEngineMsg); - if (toRuleEngineMsg.hasToRuleEngineMsg()) { - forwardToRuleEngineActor(toRuleEngineMsg.getToRuleEngineMsg(), callback); + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); } else { callback.onSuccess(); } @@ -122,16 +128,16 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } - //TODO 2.5 - private void forwardToRuleEngineActor(TransportProtos.TransportToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - log.info("Received RULE ENGINE msg: {}", toRuleEngineMsg); + private void forwardToRuleEngineActor(TenantId tenantId, ByteString tbMsgData, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(tbMsgData.toByteArray(), callback); + log.info("[{}] Received RULE ENGINE msg: {}", tbMsg.getType(), tbMsg); + actorContext.getAppActor().tell(new QueueToRuleEngineMsg(tenantId, tbMsg), ActorRef.noSender()); // if (statsEnabled) { // stats.log(toDeviceActorMsg); // } -// actorContext.getAppActor().tell(new TransportToDeviceActorMsgWrapper(toDeviceActorMsg, callback), ActorRef.noSender()); } - @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") + @Scheduled(fixedDelayString = "${queue.rule_engine.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { stats.printStats(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index d275adc4be..d616bbef16 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java index 8baecbf37a..d8f12b8f0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java @@ -39,7 +39,7 @@ import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.gen.cluster.ClusterAPIProtos; @@ -185,8 +185,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService { try { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON , json.writeValueAsString(entityNode) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg))); + , null, null, null); + actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new QueueToRuleEngineMsg(msg.getTenantId(), tbMsg))); } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java index 73bdd27b78..1454b0c6fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java @@ -94,7 +94,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S String newData = data != null ? data : msg.getData(); TbMsgMetaData newMetadata = metadata != null ? new TbMsgMetaData(metadata) : msg.getMetaData().copy(); String newMessageType = !StringUtils.isEmpty(messageType) ? messageType : msg.getType(); - return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + return new TbMsg(msg.getId(), newMessageType, msg.getOriginator(), newMetadata, msg.getDataType(), newData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getCallback()); } catch (Throwable th) { th.printStackTrace(); throw new RuntimeException("Failed to unbind message data from javascript result", th); diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 51f544712e..816479b1e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -50,7 +50,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; @@ -61,7 +61,7 @@ import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -501,8 +501,8 @@ public class DefaultDeviceStateService implements DeviceStateService { try { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state) - , null, null, 0L); - actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg))); + , null, null, null); + actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new QueueToRuleEngineMsg(stateData.getTenantId(), tbMsg))); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 14a25c2e9d..430dd77a36 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; /** * Created by ashvayka on 01.05.18. diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java index f9d60b37a5..89f745260a 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index 267ea14ac1..be186e0ddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -49,7 +48,7 @@ import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java index a80fb0db92..a71155f0ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java @@ -17,7 +17,7 @@ package org.thingsboard.server.service.subscription; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; public interface LocalSubscriptionService { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index bb236141f8..d4d9fa5841 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d94eb3f2b2..58ed13c1e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -42,7 +42,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 173c293363..b9424d49d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -113,7 +113,7 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ @Override public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { - endLocalTransaction(TbMsg.fromBytes(data), msg -> { + endLocalTransaction(TbMsg.fromBytes(data, null), msg -> { }, error -> { }); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 880d7507ce..16e0dd67f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.service.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; import java.util.UUID; diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 6bb1f4d99d..265a552efb 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -16,9 +16,6 @@ package org.thingsboard.server.rules.flow; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Lists; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; @@ -30,25 +27,22 @@ import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.rule.RuleChainService; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.Predicate; import java.util.stream.Collectors; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -154,9 +148,9 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), "CUSTOM", device.getId(), - new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + new TbMsgMetaData(), TbMsgDataType.JSON, + "{}", null, null, null); + actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); @@ -270,8 +264,9 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule "CUSTOM", device.getId(), new TbMsgMetaData(), - "{}", null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + TbMsgDataType.JSON, + "{}", null, null, null); + actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 5f806d1ea0..a4360d51e9 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.rules.lifecycle; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; @@ -38,13 +37,13 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -141,9 +140,10 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac "CUSTOM", device.getId(), new TbMsgMetaData(), + TbMsgDataType.JSON, "{}", - null, null, 0L); - actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg))); + null, null, null); + actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 82b3df9736..3c5f86a3fc 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -24,6 +24,7 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -62,7 +63,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -78,7 +79,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -94,7 +95,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -112,7 +113,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -126,7 +127,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData,TbMsgDataType.JSON, rawJson, null, null, null); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -147,7 +148,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -169,7 +170,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, null, null, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 04b0cae56b..4076552863 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.msg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; + /** * Created by ashvayka on 15.03.18. */ @@ -43,11 +45,11 @@ public enum MsgType { COMPONENT_LIFE_CYCLE_MSG, /** - * Misc messages from the REST API/SERVICE layer to the new rule engine. + * Misc messages consumed from the Queue and forwarded to Rule Engine Actor. * - * See {@link org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg} + * See {@link QueueToRuleEngineMsg} */ - SERVICE_TO_RULE_ENGINE_MSG, + QUEUE_TO_RULE_ENGINE_MSG, /** * Message that is sent by RuleChainActor to RuleActor with command to process TbMsg. diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 4e7090f934..8dc16a6776 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -17,12 +17,14 @@ package org.thingsboard.server.common.msg; import com.google.protobuf.InvalidProtocolBufferException; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; import java.nio.ByteBuffer; @@ -32,6 +34,7 @@ import java.util.UUID; * Created by ashvayka on 13.01.18. */ @Data +@Builder @AllArgsConstructor public final class TbMsg implements Serializable { @@ -42,29 +45,14 @@ public final class TbMsg implements Serializable { private final TbMsgDataType dataType; private final String data; private final TbMsgTransactionData transactionData; - - //The following fields are not persisted to DB, because they can always be recovered from the context; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; - private final long clusterPartition; - - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - this.id = id; - this.type = type; - this.originator = originator; - this.metaData = metaData; - this.data = data; - this.dataType = TbMsgDataType.JSON; - this.transactionData = new TbMsgTransactionData(id, originator); - this.ruleChainId = ruleChainId; - this.ruleNodeId = ruleNodeId; - this.clusterPartition = clusterPartition; - } + //This field is not serialized because we use queues and there is no need to do it + private final TbMsgCallback callback; public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, clusterPartition); + RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); } public static byte[] toByteArray(TbMsg msg) { @@ -105,36 +93,31 @@ public final class TbMsg implements Serializable { } - public static ByteBuffer toBytes(TbMsg msg) { - return ByteBuffer.wrap(toByteArray(msg)); - } - - public static TbMsg fromBytes(byte[] data) { - return fromBytes(ByteBuffer.wrap(data)); - } - - public static TbMsg fromBytes(ByteBuffer buffer) { + public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { try { - MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(buffer.array()); + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - RuleChainId ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + RuleChainId ruleChainId = null; RuleNodeId ruleNodeId = null; + if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) { + ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB())); + } if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, proto.getClusterPartition()); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } - public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, long clusterPartition) { - return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, clusterPartition); + public TbMsg copy(UUID newId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + return new TbMsg(newId, type, originator, metaData.copy(), dataType, data, transactionData, ruleChainId, ruleNodeId, callback); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java similarity index 85% rename from common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index a717af85a4..6f93301a48 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/system/ServiceToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.system; +package org.thingsboard.server.common.msg.queue; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; @@ -27,13 +27,13 @@ import java.io.Serializable; * Created by ashvayka on 15.03.18. */ @Data -public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable { +public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; @Override public MsgType getMsgType() { - return MsgType.SERVICE_TO_RULE_ENGINE_MSG; + return MsgType.QUEUE_TO_RULE_ENGINE_MSG; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java index 85fec1d2ab..4f0e383fe9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.common.msg.queue; public interface TbMsgCallback { diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 19e8d64e87..18c47e542b 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -226,7 +226,6 @@ message TransportToRuleEngineMsg { PostAttributeMsg postAttributes = 3; ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 4; ToServerRpcRequestMsg toServerRPCCallRequest = 5; - SubscriptionInfoProto subscriptionInfo = 6; } message DeviceActorToTransportMsg { @@ -365,7 +364,9 @@ message ToCoreMsg { /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { - TransportToRuleEngineMsg toRuleEngineMsg = 1; + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + bytes tbMsg = 3; } /* Messages that are handled by ThingsBoard Transport Service */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index d8d827aa52..1a80aa4a6e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -15,11 +15,20 @@ */ package org.thingsboard.server.common.transport.service; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsgMetadata; @@ -50,16 +59,19 @@ import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Created by ashvayka on 17.10.18. @@ -82,6 +94,7 @@ public class DefaultTransportService implements TransportService { @Value("${queue.transport.poll_interval}") private int notificationsPollDuration; + private final Gson gson = new Gson(); private final TransportQueueProvider queueProvider; private final PartitionService partitionService; @@ -221,8 +234,19 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). - setPostTelemetry(msg).build(), callback); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + MsgPackCallback packCallback = new MsgPackCallback(msg.getTsKvListCount(), callback); + for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("ts", tsKv.getTs() + ""); + JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); + TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), + deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); + sendToRuleEngine(tenantId, tbMsg, packCallback); + } } } @@ -230,8 +254,15 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). - setPostAttributes(msg).build(), callback); + TenantId tenantId = new TenantId(new UUID(sessionInfo.getTenantIdMSB(), sessionInfo.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); + JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, + TbMsgDataType.JSON, gson.toJson(json), null, null, null); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); } } @@ -278,8 +309,8 @@ public class DefaultTransportService implements TransportService { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); - sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). - setToServerRPCCallRequest(msg).build(), callback); +// sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). +// setToServerRPCCallRequest(msg).build(), new TransportTbQueueCallback(callback)); } } @@ -455,12 +486,12 @@ public class DefaultTransportService implements TransportService { new TransportTbQueueCallback(callback) : null); } - protected void sendToRuleEngine(TransportProtos.SessionInfoProto sessionInfo, TransportToRuleEngineMsg toRuleEngineMsg, TransportServiceCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); - ruleEngineMsgProducer.send(tpi, - new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), - ToRuleEngineMsg.newBuilder().setToRuleEngineMsg(toRuleEngineMsg).build()), callback != null ? - new TransportTbQueueCallback(callback) : null); + protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(ByteString.copyFrom(TbMsg.toByteArray(tbMsg))) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); } private class TransportTbQueueCallback implements TbQueueCallback { @@ -480,4 +511,26 @@ public class DefaultTransportService implements TransportService { DefaultTransportService.this.transportCallbackExecutor.submit(() -> callback.onError(t)); } } + + private class MsgPackCallback implements TbQueueCallback { + private final AtomicInteger msgCount; + private final TransportServiceCallback callback; + + public MsgPackCallback(Integer msgCount, TransportServiceCallback callback) { + this.msgCount = new AtomicInteger(msgCount); + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (msgCount.decrementAndGet() <= 0) { + callback.onSuccess(null); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onError(t); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonUtils.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java index 5621362c09..583b73dba1 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonUtils.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/JsonUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.transport.util; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 2e99e75b53..1952985e53 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -75,7 +75,8 @@ public class TbMsgCountNode implements TbNode { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L); + //TODO 2.5: Callback? + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, null); ctx.tellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java index cff9faaab5..3349706221 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java @@ -66,7 +66,8 @@ class TbRedisQueueProcessor { if (listOperations.size(redisKey) > 0) { List list = listOperations.range(redisKey, -10, -1); list.forEach(obj -> { - TbMsg msg = TbMsg.fromBytes((byte[]) obj); + //TODO 2.5: Callback? + TbMsg msg = TbMsg.fromBytes((byte[]) obj, null); log.debug("Trying to send the message: {}", msg); listOperations.remove(redisKey, -1, obj); httpClient.processMessage(ctx, msg, this); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 5423236499..a6bd7b6164 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -57,8 +57,9 @@ public class TbSynchronizationBeginNode implements TbNode { log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); + //TODO 2.5: Callback? TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, - msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), msg.getClusterPartition()); + msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), null); ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 67c5b1cc18..e0b76f1cda 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -37,7 +37,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; @@ -99,7 +101,7 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -141,7 +143,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -164,7 +166,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -209,7 +211,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -256,7 +258,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index 27feed93ba..c840e708eb 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -58,7 +59,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -71,7 +72,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -84,7 +85,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 1e752b55b8..4934e14062 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -65,7 +66,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index ace02bacd7..8315d8e40b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.io.IOException; @@ -62,7 +63,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 0af1cef8ae..0169cadcb2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.*; import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -102,7 +103,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -127,7 +128,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -152,7 +153,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -166,7 +167,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); entityAttributeFetched(customerId); } @@ -177,7 +178,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -192,7 +193,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -207,7 +208,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -234,7 +235,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 4c1edce8f6..1ed66fc786 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.asset.AssetService; @@ -91,7 +92,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -119,7 +120,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -146,7 +147,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), "{}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index aa9ad8b76f..9a315f893b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -31,6 +31,7 @@ import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; @@ -62,8 +63,8 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); - TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, "{new}", ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId, null); + TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); @@ -84,7 +85,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, rawJson, ruleChainId, ruleNodeId, 0L); + TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); From 2ccce3b6d9c3d65382d2d3db7bae66861bbaad9a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Mar 2020 14:08:21 +0200 Subject: [PATCH 021/196] Main Logic of RuleChainActor to handle queue messages --- .../server/actors/ActorSystemContext.java | 10 + .../device/DeviceActorMessageProcessor.java | 28 +-- .../actors/ruleChain/DefaultTbContext.java | 8 +- .../actors/ruleChain/RuleChainActor.java | 6 +- .../RuleChainActorMessageProcessor.java | 214 +++++++++--------- .../ruleChain/RuleChainToRuleChainMsg.java | 1 - .../server/actors/tenant/TenantActor.java | 12 - .../DefaultTbRuleEngineConsumerService.java | 3 +- .../server/common/msg/MsgType.java | 1 - .../thingsboard/server/common/msg/TbMsg.java | 30 ++- .../MultipleTbQueueTbMsgCallbackWrapper.java | 43 ++++ .../queue/TbQueueTbMsgCallbackWrapper.java | 32 +-- .../ConsistentHashPartitionService.java | 31 ++- .../queue/discovery/PartitionService.java | 3 +- .../queue/discovery/TopicPartitionInfo.java | 12 +- .../discovery/TopicPartitionInfoKey.java | 43 ++++ .../queue/kafka/TBKafkaConsumerTemplate.java | 2 +- 17 files changed, 292 insertions(+), 187 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java rename application/src/main/java/org/thingsboard/server/actors/device/DeviceActorToRuleEngineMsg.java => common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java (52%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 46e6df60ab..e5158377cd 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -63,7 +63,9 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -151,6 +153,14 @@ public class ActorSystemContext { @Getter private RuleChainService ruleChainService; + @Autowired + @Getter + private PartitionService partitionService; + + @Autowired + @Getter + private TbRuleEngineQueueProvider ruleEngineQueueProvider; + @Autowired @Getter private TimeseriesService tsService; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 1844539b2c..4713662067 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -331,18 +331,18 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { - UUID sessionId = getSessionId(sessionInfo); - JsonObject json = new JsonObject(); - json.addProperty("method", request.getMethodName()); - json.add("params", JsonUtils.parse(request.getParams())); - - TbMsgMetaData requestMetaData = defaultMetaData.copy(); - requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - - scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); - toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); +// UUID sessionId = getSessionId(sessionInfo); +// JsonObject json = new JsonObject(); +// json.addProperty("method", request.getMethodName()); +// json.add("params", JsonUtils.parse(request.getParams())); +// +// TbMsgMetaData requestMetaData = defaultMetaData.copy(); +// requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); +// TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); +// context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); +// +// scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); +// toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); } private TransportProtos.SessionType getSessionType(UUID sessionId) { @@ -372,10 +372,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void pushToRuleEngine(ActorContext context, TbMsg tbMsg) { - context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); - } - void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { if (attributeSubscriptions.size() > 0) { boolean hasNotificationData = false; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 337a234e2d..a5b5254174 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -65,6 +65,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -118,10 +119,7 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - //TODO 2.5 -// Optional address = mainCtx.getRoutingService().resolveById(entityId); -// return !address.isPresent(); - return true; + return mainCtx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -143,7 +141,7 @@ class DefaultTbContext implements TbContext { @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(),null); + return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), null); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 76f831be98..83cde28a45 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.OneForOneStrategy; import akka.actor.SupervisorStrategy; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg; import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; @@ -43,10 +42,7 @@ public class RuleChainActor extends ComponentActor { - private static final long DEFAULT_CLUSTER_PARTITION = 0L; private final ActorRef parent; private final ActorRef self; private final Map nodeActors; private final Map> nodeRoutes; private final RuleChainService service; + private final PartitionService partitionService; + private final TbQueueProducer> producer; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - private String ruleChainName; RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { @@ -76,7 +83,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.ruleChainName = ruleChainId.toString(); + this.partitionService = systemContext.getPartitionService(); + this.producer = systemContext.getRuleEngineQueueProvider().getRuleEngineMsgProducer(); } @Override @@ -89,7 +97,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); // Creating and starting the actors; @@ -110,7 +117,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); for (RuleNode ruleNode : ruleNodeList) { @@ -189,101 +195,113 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor address = systemContext.getRoutingService().resolveById(originatorEntityId); -// if (address.isPresent()) { -// onRemoteTellNext(address.get(), envelope); -// } else { - onLocalTellNext(envelope); -// } + try { + checkActive(); + EntityId entityId = msg.getOriginator(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + RuleNodeId originatorNodeId = envelope.getOriginator(); + List relations = nodeRoutes.get(originatorNodeId).stream() + .filter(r -> contains(envelope.getRelationTypes(), r.getType())) + .collect(Collectors.toList()); + int relationsCount = relations.size(); + if (relationsCount == 0) { + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); + //TODO 2.5: Maybe let's check that the output relation is not a Failure? + if (envelope.getRelationTypes().contains(TbRelationTypes.FAILURE)) { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, envelope.getOriginator().getId()); + msg.getCallback().onFailure(new RuntimeException("Failure during message processing by Rule Node [" + envelope.getOriginator().getId().toString() + "]")); + } else { + msg.getCallback().onSuccess(); + } + } else if (relationsCount == 1) { + for (RuleNodeRelation relation : relations) { + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); + pushToTarget(tpi, msg, relation.getOut(), relation.getType()); + } + } else { + MultipleTbQueueTbMsgCallbackWrapper callbackWrapper = new MultipleTbQueueTbMsgCallbackWrapper(relationsCount, msg.getCallback()); + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relations); + for (RuleNodeRelation relation : relations) { + EntityId target = relation.getOut(); + putToQueue(tpi, msg, callbackWrapper, target); + } + } + } catch (Exception e) { + msg.getCallback().onFailure(e); + } } - private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator()); - envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId); - //TODO 2.5 -// systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope)); + private void putToQueue(TopicPartitionInfo tpi, TbMsg msg, TbQueueCallback callbackWrapper, EntityId target) { + switch (target.getEntityType()) { + case RULE_NODE: + putToQueue(tpi, msg.copyWithRuleNodeId(entityId, new RuleNodeId(target.getId())), callbackWrapper); + break; + case RULE_CHAIN: + putToQueue(tpi, msg.copyWithRuleChainId(new RuleChainId(target.getId())), callbackWrapper); + break; + } } - private void onLocalTellNext(RuleNodeToRuleChainTellNextMsg envelope) { - TbMsg msg = envelope.getMsg(); - RuleNodeId originatorNodeId = envelope.getOriginator(); - List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) - .collect(Collectors.toList()); - int relationsCount = relations.size(); - EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId(); - if (relationsCount == 0) { - log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } - } else if (relationsCount == 1) { - for (RuleNodeRelation relation : relations) { - log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - pushToTarget(msg, relation.getOut(), relation.getType()); + private void pushToTarget(TopicPartitionInfo tpi, TbMsg msg, EntityId target, String fromRelationType) { + if (tpi.isMyPartition()) { + switch (target.getEntityType()) { + case RULE_NODE: + pushMsgToNode(nodeActors.get(new RuleNodeId(target.getId())), msg, fromRelationType); + break; + case RULE_CHAIN: + parent.tell(new RuleChainToRuleChainMsg(new RuleChainId(target.getId()), entityId, msg, fromRelationType), self); + break; } } else { - for (RuleNodeRelation relation : relations) { - EntityId target = relation.getOut(); - log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut()); - switch (target.getEntityType()) { - case RULE_NODE: - enqueueAndForwardMsgCopyToNode(msg, target, relation.getType()); - break; - case RULE_CHAIN: - enqueueAndForwardMsgCopyToChain(msg, target, relation.getType()); - break; - } - } - //TODO: Ideally this should happen in async way when all targets confirm that the copied messages are successfully written to corresponding target queues. - if (ackId != null) { -// TODO: Ack this message in Kafka -// queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition()); - } + putToQueue(tpi, msg, new TbQueueTbMsgCallbackWrapper(msg.getCallback()), target); } } + private void putToQueue(TopicPartitionInfo tpi, TbMsg newMsg, TbQueueCallback callbackWrapper) { + ToRuleEngineMsg toQueueMsg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(ByteString.copyFrom(TbMsg.toByteArray(newMsg))) + .build(); + producer.send(tpi, new TbProtoQueueMsg<>(newMsg.getId(), toQueueMsg), callbackWrapper); + } + private boolean contains(Set relationTypes, String type) { if (relationTypes == null) { return true; @@ -296,38 +314,10 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor> myPartitions = new ConcurrentHashMap<>(); //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); + private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); private Map tbCoreNotificationTopics = new HashMap<>(); private Map tbRuleEngineNotificationTopics = new HashMap<>(); @@ -87,12 +89,12 @@ public class ConsistentHashPartitionService implements PartitionService { } @Override - public List getCurrentPartitions(ServiceType serviceType) { + public Set getCurrentPartitions(ServiceType serviceType) { ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); TenantId tenantId = getSystemOrIsolatedTenantId(currentService); ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); List partitions = myPartitions.get(serviceKey); - List topicPartitions = new ArrayList<>(); + Set topicPartitions = new LinkedHashSet<>(); for (Integer partition : partitions) { TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); @@ -112,7 +114,9 @@ public class ConsistentHashPartitionService implements PartitionService { .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); int partition = Math.abs(hash % partitionSizes.get(serviceType)); - return buildTopicPartitionInfo(serviceType, tenantId, partition); + boolean isolatedTenant = isIsolated(serviceType, tenantId); + TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceType, isolatedTenant ? tenantId : null, partition); + return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceType, tenantId, partition)); } @Override @@ -156,8 +160,8 @@ public class ConsistentHashPartitionService implements PartitionService { tpiList.add(getNotificationsTopic(serviceKey.getServiceType(), serviceInfoProvider.getServiceId())); applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } - }); + tpiCache.clear(); if (currentOtherServices == null) { currentOtherServices = new ArrayList<>(otherServices); @@ -207,7 +211,7 @@ public class ConsistentHashPartitionService implements PartitionService { } private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null); + return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null, false); } private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { @@ -215,16 +219,29 @@ public class ConsistentHashPartitionService implements PartitionService { } private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { - boolean isolated = isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); tpi.topic(partitionTopics.get(serviceType)); tpi.partition(partition); - if (isolated) { + ServiceKey myPartitionsSearchKey; + if (isIsolated(serviceType, tenantId)) { tpi.tenantId(tenantId); + myPartitionsSearchKey = new ServiceKey(serviceType, tenantId); + } else { + myPartitionsSearchKey = new ServiceKey(serviceType, new TenantId(TenantId.NULL_UUID)); + } + List partitions = myPartitions.get(myPartitionsSearchKey); + if (partitions != null) { + tpi.myPartition(partitions.contains(partition)); + } else { + tpi.myPartition(false); } return tpi.build(); } + private boolean isIsolated(ServiceType serviceType, TenantId tenantId) { + return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + } + private void logServiceInfo(TransportProtos.ServiceInfo server) { TenantId tenantId = getSystemOrIsolatedTenantId(server); if (tenantId.isNullUid()) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 46864af79f..4eb4ca9a69 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -20,13 +20,14 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; +import java.util.Set; /** * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} */ public interface PartitionService { - List getCurrentPartitions(ServiceType serviceType); + Set getCurrentPartitions(ServiceType serviceType); TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java index 05c5c1586d..1164465ed7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java @@ -16,6 +16,7 @@ package org.thingsboard.server.queue.discovery; import lombok.Builder; +import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; @@ -26,13 +27,17 @@ public class TopicPartitionInfo { private final String topic; private final TenantId tenantId; private final Integer partition; + @Getter private final String fullTopicName; + @Getter + private final boolean myPartition; @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition) { + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { this.topic = topic; this.tenantId = tenantId; this.partition = partition; + this.myPartition = myPartition; String tmp = topic; if (tenantId != null) { tmp += "." + tenantId.getId().toString(); @@ -40,7 +45,6 @@ public class TopicPartitionInfo { if (partition != null) { tmp += "." + partition; } - this.fullTopicName = tmp; } @@ -56,10 +60,6 @@ public class TopicPartitionInfo { return Optional.ofNullable(partition); } - public String getFullTopicName() { - return fullTopicName; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java new file mode 100644 index 0000000000..4d02647fd0 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.AllArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Objects; + +@AllArgsConstructor +public class TopicPartitionInfoKey { + private ServiceType serviceType; + private TenantId isolatedTenantId; + private int partition; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; + return partition == that.partition && + serviceType == that.serviceType && + Objects.equals(isolatedTenantId, that.isolatedTenantId); + } + + @Override + public int hashCode() { + return Objects.hash(serviceType, isolatedTenantId, partition); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index 52086b3058..ec08d69be7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -75,7 +75,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null)); + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); subscribed = false; } From 39ff12fea27db251d2329cd071b8a75b4bb36394 Mon Sep 17 00:00:00 2001 From: blackstar-baba <535650957@qq.com> Date: Tue, 24 Mar 2020 21:49:30 +0800 Subject: [PATCH 022/196] fix bug: When using jwt token to access the api, JwtTokenAuthenticationProcessingFilter is executed 2 times (#2531) --- .../server/config/ThingsboardSecurityConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 526f0b73c8..53876f4040 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -107,7 +107,6 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt return filter; } - @Bean protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception { List pathsToSkip = new ArrayList(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS)); pathsToSkip.addAll(Arrays.asList(WS_TOKEN_BASED_AUTH_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT, From 08c8d935685c1e4fc72556e7ce5741a5fc63a1c0 Mon Sep 17 00:00:00 2001 From: Oleg Kolesnik <31017535+jktu2870@users.noreply.github.com> Date: Tue, 24 Mar 2020 15:50:52 +0200 Subject: [PATCH 023/196] UI:fix. Map widget2 (#2413) preserves previous data on map if no data for current date range received --- ui/src/app/widget/lib/map-widget2.js | 78 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/ui/src/app/widget/lib/map-widget2.js b/ui/src/app/widget/lib/map-widget2.js index bf300630cc..3d3fd4a38d 100644 --- a/ui/src/app/widget/lib/map-widget2.js +++ b/ui/src/app/widget/lib/map-widget2.js @@ -584,51 +584,53 @@ export default class TbMapWidgetV2 { var latData = data[location.latIndex].data; var lngData = data[location.lngIndex].data; var lat, lng, latLng; - if (latData.length > 0 && lngData.length > 0) { - if (tbMap.drawRoutes) { - // Create or update route - var latLngs = []; - for (var i = 0; i < latData.length; i++) { - lat = latData[i][1]; - lng = lngData[i][1]; - if (angular.isDefined(lat) && lat != null && angular.isDefined(lng) && lng != null) { - latLng = tbMap.map.createLatLng(lat, lng); - if (i == 0 || !latLngs[latLngs.length - 1].equals(latLng)) { - latLngs.push(latLng); - } - } - } - if (latLngs.length > 0) { - var markerLocation = latLngs[latLngs.length - 1]; - createOrUpdateLocationMarker(location, markerLocation, dataMap); - } - if (!location.polyline) { - location.polyline = tbMap.map.createPolyline(latLngs, location.settings); - tbMap.polylines.push(location.polyline); - locationChanged = true; - } else { - var prevPath = tbMap.map.getPolylineLatLngs(location.polyline); - if (!prevPath || !arraysEqual(prevPath, latLngs)) { - tbMap.map.setPolylineLatLngs(location.polyline, latLngs); - locationChanged = true; - } - } - } else { - // Create or update marker - lat = latData[latData.length - 1][1]; - lng = lngData[lngData.length - 1][1]; + + if (tbMap.drawRoutes) { + // Create or update route + var latLngs = []; + for (var i = 0; i < latData.length; i++) { + lat = latData[i][1]; + lng = lngData[i][1]; if (angular.isDefined(lat) && lat != null && angular.isDefined(lng) && lng != null) { latLng = tbMap.map.createLatLng(lat, lng); - if (createOrUpdateLocationMarker(location, latLng, dataMap)) { - locationChanged = true; + if (i == 0 || !latLngs[latLngs.length - 1].equals(latLng)) { + latLngs.push(latLng); } } } - if (location.marker) { - updateLocationStyle(location, dataMap); + if (latLngs.length > 0) { + var markerLocation = latLngs[latLngs.length - 1]; + createOrUpdateLocationMarker(location, markerLocation, dataMap); + } else if (location.marker) { + tbMap.map.removeMarker(location.marker); + delete location.marker; + } + if (!location.polyline) { + location.polyline = tbMap.map.createPolyline(latLngs, location.settings); + tbMap.polylines.push(location.polyline); + locationChanged = true; + } else { + var prevPath = tbMap.map.getPolylineLatLngs(location.polyline); + if (!prevPath || !arraysEqual(prevPath, latLngs)) { + tbMap.map.setPolylineLatLngs(location.polyline, latLngs); + locationChanged = true; + } + } + } else { + // Create or update marker + lat = latData[latData.length - 1][1]; + lng = lngData[lngData.length - 1][1]; + if (angular.isDefined(lat) && lat != null && angular.isDefined(lng) && lng != null) { + latLng = tbMap.map.createLatLng(lat, lng); + if (createOrUpdateLocationMarker(location, latLng, dataMap)) { + locationChanged = true; + } } - } + if (location.marker) { + updateLocationStyle(location, dataMap); + } + } return locationChanged; } From cb2278ed56d00c6114d1cf43d629f10d3ee2f5d8 Mon Sep 17 00:00:00 2001 From: Chantsova Ekaterina Date: Tue, 24 Mar 2020 15:53:18 +0200 Subject: [PATCH 024/196] Remove empty strings from Spanish translations (#2539) --- ui/src/app/locale/locale.constant-es_ES.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/src/app/locale/locale.constant-es_ES.json b/ui/src/app/locale/locale.constant-es_ES.json index 69d28426e3..8394e9ed65 100644 --- a/ui/src/app/locale/locale.constant-es_ES.json +++ b/ui/src/app/locale/locale.constant-es_ES.json @@ -1226,7 +1226,6 @@ "remember-me": "Recordarme", "forgot-password": "¿Olvidó la contraseña?", "password-reset": "Restablecer contraseña", - "expired-password-reset-message": "", "new-password": "Nueva contraseña", "new-password-again": "Repita la nueva contraseña", "password-link-sent-message": "¡El enlace para el restablecer la contraseña fue enviado correctamente!", @@ -1556,7 +1555,6 @@ "export": "Exportar widget" }, "widget-action": { - "custom-pretty": "", "header-button": "Botón del encabezado del widget", "open-dashboard-state": "Navegar a nuevo estado del panel", "update-dashboard-state": "Actualizar estado vigente del panel", From 07bdcac0fe2b8b5165cd7823fa53deaba26ea45a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Mar 2020 19:18:27 +0200 Subject: [PATCH 025/196] Refactoring of Rule Engine API --- .../server/actors/ActorSystemContext.java | 19 ++- .../actors/ruleChain/DefaultTbContext.java | 16 ++- .../RuleChainActorMessageProcessor.java | 10 +- .../server/controller/BaseController.java | 19 ++- .../queue/DefaultTbCoreConsumerService.java | 16 ++- .../DefaultTbRuleEngineConsumerService.java | 65 +++++++--- .../server/service/queue/MsgPackCallback.java | 28 +++-- .../TbRuleEngineProcessingDecision.java | 16 +++ .../TbRuleEngineProcessingResult.java | 33 +++++ .../TbRuleEngineProcessingStrategy.java | 7 ++ ...TbRuleEngineProcessingStrategyFactory.java | 118 ++++++++++++++++++ .../state/DefaultDeviceStateService.java | 10 +- .../DefaultSubscriptionManagerService.java | 2 +- .../SubscriptionManagerService.java | 3 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../src/main/resources/thingsboard.yml | 74 ++++++----- .../thingsboard/server/common/msg/TbMsg.java | 6 +- .../service/DefaultTransportService.java | 8 +- 18 files changed, 362 insertions(+), 90 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e5158377cd..64a713b336 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -63,8 +63,13 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; @@ -154,7 +159,6 @@ public class ActorSystemContext { private RuleChainService ruleChainService; @Autowired - @Getter private PartitionService partitionService; @Autowired @@ -402,6 +406,17 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } + public TbQueueProducer> getTbCoreMsgProducer() { + return ruleEngineQueueProvider.getTbCoreMsgProducer(); + } + + public TbQueueProducer> getRuleEngineMsgProducer() { + return ruleEngineQueueProvider.getRuleEngineMsgProducer(); + } + + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, tenantId, entityId); + } public String getServerAddress() { return serviceInfoProvider.getServiceId(); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index a5b5254174..c9273e0a11 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -65,7 +65,10 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -119,7 +122,7 @@ class DefaultTbContext implements TbContext { @Override public boolean isLocalEntity(EntityId entityId) { - return mainCtx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); + return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); } private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) { @@ -151,8 +154,13 @@ class DefaultTbContext implements TbContext { } @Override - public void sendTbMsgToRuleEngine(TbMsg msg) { - mainCtx.getActorService().onMsg(new SendToClusterMsg(msg.getOriginator(), new QueueToRuleEngineMsg(getTenantId(), msg))); + public void sendTbMsgToRuleEngine(TbMsg tbMsg) { + TenantId tenantId = getTenantId(); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); + mainCtx.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index b3c71b3b62..13d370e037 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -68,7 +68,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor nodeActors; private final Map> nodeRoutes; private final RuleChainService service; - private final PartitionService partitionService; private final TbQueueProducer> producer; private RuleNodeId firstId; @@ -83,7 +82,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.partitionService = systemContext.getPartitionService(); this.producer = systemContext.getRuleEngineQueueProvider().getRuleEngineMsgProducer(); } @@ -234,7 +232,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relations = nodeRoutes.get(originatorNodeId).stream() .filter(r -> contains(envelope.getRelationTypes(), r.getType())) @@ -242,9 +240,9 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(newMsg.getId(), toQueueMsg), callbackWrapper); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index e09b8ca673..bc88544243 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -93,6 +93,12 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -185,6 +191,12 @@ public abstract class BaseController { @Autowired protected ClaimDevicesService claimDevicesService; + @Autowired + protected PartitionService partitionService; + + @Autowired + protected TbCoreQueueProvider coreQueueProvider; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -662,7 +674,12 @@ public abstract class BaseController { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON , json.writeValueAsString(entityNode) , null, null, null); - actorService.onMsg(new SendToClusterMsg(entityId, new QueueToRuleEngineMsg(user.getTenantId(), tbMsg))); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, user.getTenantId(), entityId); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(user.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(user.getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + coreQueueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c081fc526a..30148dfe08 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,7 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -104,11 +105,13 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { if (msgs.isEmpty()) { continue; } - ConcurrentMap> ackMap = msgs.stream().collect( + ConcurrentMap> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> successMap = new ConcurrentHashMap<>(); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - ackMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); + pendingMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, successMap, failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToDeviceActorMsg()) { @@ -130,7 +133,8 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } }); if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } catch (Exception e) { @@ -182,7 +186,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); } else if (msg.hasTsUpdate()) { TransportProtos.TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); - subscriptionManagerService.onTimeseriesDataUpdate( + subscriptionManagerService.onTimeSeriesUpdate( new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 1f84c7cc98..986d858043 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -36,11 +36,16 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -64,10 +69,12 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS private final ActorSystemContext actorContext; private final TbQueueConsumer> consumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); + private final TbRuleEngineProcessingStrategyFactory factory; private volatile ExecutorService mainConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + this.factory = factory; this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); this.actorContext = actorContext; } @@ -75,6 +82,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @PostConstruct public void init() { this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + this.factory.newInstance(); } @Override @@ -94,29 +102,46 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS if (msgs.isEmpty()) { continue; } - ConcurrentMap> ackMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - ackMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, ackMap); - try { - TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); - } else { - callback.onSuccess(); + TbRuleEngineProcessingStrategy strategy = factory.newInstance(); + TbRuleEngineProcessingDecision decision = null; + boolean firstAttempt = true; + while (!stopped && (firstAttempt || !decision.isCommit())) { + ConcurrentMap> allMap; + if (firstAttempt) { + allMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + firstAttempt = false; + } else { + allMap = decision.getReprocessMap(); + } + ConcurrentMap> successMap = new ConcurrentHashMap<>(); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + allMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); + try { + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + callback.onFailure(e); } - } catch (Throwable e) { - callback.onFailure(e); + }); + + boolean timeout = false; + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + timeout = true; } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - ackMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); } consumer.commit(); } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); + log.warn("Failed to process messages from queue.", e); try { Thread.sleep(pollDuration); } catch (InterruptedException e2) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index d616bbef16..420c61d5d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,25 +20,37 @@ import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @Slf4j -public class MsgPackCallback implements TbMsgCallback { +public class MsgPackCallback implements TbMsgCallback { private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap> ackMap; + private final ConcurrentMap ackMap; + private final ConcurrentMap successMap; + private final ConcurrentMap failedMap; private final UUID id; - public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, ConcurrentMap> ackMap) { + public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap successMap, + ConcurrentMap failedMap) { this.id = id; this.processingTimeoutLatch = processingTimeoutLatch; this.ackMap = ackMap; + this.successMap = successMap; + this.failedMap = failedMap; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - if (ackMap.remove(id) != null && ackMap.isEmpty()) { + T msg = ackMap.remove(id); + if (msg != null) { + successMap.put(id, msg); + } + if (msg != null && ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } } @@ -46,8 +58,10 @@ public class MsgPackCallback i @Override public void onFailure(Throwable t) { log.trace("[{}] ON FAILURE", id); - TbProtoQueueMsg message = ackMap.remove(id); - log.warn("Failed to process message: {}", message.getValue(), t); + T msg = ackMap.remove(id); + if (msg != null) { + failedMap.put(id, msg); + } if (ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java new file mode 100644 index 0000000000..b116fd7443 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -0,0 +1,16 @@ +package org.thingsboard.server.service.queue.processing; + +import lombok.Data; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +@Data +public class TbRuleEngineProcessingDecision { + + private final boolean commit; + private final ConcurrentMap> reprocessMap; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java new file mode 100644 index 0000000000..601a3e9a9b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -0,0 +1,33 @@ +package org.thingsboard.server.service.queue.processing; + +import lombok.Getter; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +public class TbRuleEngineProcessingResult { + + @Getter + private boolean success; + @Getter + private boolean timeout; + @Getter + private ConcurrentMap> pendingMap; + @Getter + private ConcurrentMap> successMap; + @Getter + private ConcurrentMap> failureMap; + + public TbRuleEngineProcessingResult(boolean timeout, + ConcurrentMap> pendingMap, + ConcurrentMap> successMap, + ConcurrentMap> failureMap) { + this.timeout = timeout; + this.pendingMap = pendingMap; + this.successMap = successMap; + this.failureMap = failureMap; + this.success = !timeout && pendingMap.isEmpty() && failureMap.isEmpty(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java new file mode 100644 index 0000000000..c644766892 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -0,0 +1,7 @@ +package org.thingsboard.server.service.queue.processing; + +public interface TbRuleEngineProcessingStrategy { + + TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java new file mode 100644 index 0000000000..336baf5360 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -0,0 +1,118 @@ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class TbRuleEngineProcessingStrategyFactory { + + @Value("${queue.rule_engine.strategy.type}") + private String strategyType; + @Value("${queue.rule_engine.strategy.retries:3}") + private int maxRetries; + @Value("${queue.rule_engine.strategy.failure_percentage:0}") + private double maxAllowedFailurePercentage; + @Value("${queue.rule_engine.strategy.pause_between_retries:3}") + private long pauseBetweenRetries; + + + public TbRuleEngineProcessingStrategy newInstance() { + switch (strategyType) { + case "SKIP_ALL": + return new SkipStrategy(); + case "RETRY_ALL": + return new RetryStrategy(true, true, true, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + case "RETRY_FAILED": + return new RetryStrategy(false, true, false, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + case "RETRY_TIMED_OUT": + return new RetryStrategy(false, false, true, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + case "RETRY_FAILED_AND_TIMED_OUT": + return new RetryStrategy(false, true, true, maxRetries, maxAllowedFailurePercentage, pauseBetweenRetries); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + strategyType + " is not supported!"); + } + } + + private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final boolean retrySuccessful; + private final boolean retryFailed; + private final boolean retryTimeout; + private final int maxRetries; + private final double maxAllowedFailurePercentage; + private final long pauseBetweenRetries; + + private int initialTotalCount; + private int retryCount; + + public RetryStrategy(boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, int maxRetries, double maxAllowedFailurePercentage, long pauseBetweenRetries) { + this.retrySuccessful = retrySuccessful; + this.retryFailed = retryFailed; + this.retryTimeout = retryTimeout; + this.maxRetries = maxRetries; + this.maxAllowedFailurePercentage = maxAllowedFailurePercentage; + this.pauseBetweenRetries = pauseBetweenRetries; + } + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + if (result.isSuccess()) { + return new TbRuleEngineProcessingDecision(true, null); + } else { + if (retryCount == 0) { + initialTotalCount = result.getPendingMap().size() + result.getFailureMap().size() + result.getSuccessMap().size(); + } + retryCount++; + double failedCount = result.getFailureMap().size() + result.getPendingMap().size(); + if (maxRetries > 0 && retryCount > maxRetries) { + log.info("Skip reprocess of the rule engine pack due to max retries"); + return new TbRuleEngineProcessingDecision(true, null); + } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { + log.info("Skip reprocess of the rule engine pack due to max allowed failure percentage"); + return new TbRuleEngineProcessingDecision(true, null); + } else { + ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); + if (retryFailed) { + result.getFailureMap().forEach(toReprocess::put); + } + if (retryTimeout) { + result.getPendingMap().forEach(toReprocess::put); + } + if (retrySuccessful) { + result.getSuccessMap().forEach(toReprocess::put); + } + log.info("Going to reprocess {} messages", toReprocess.size()); + //TODO: 2.5 Log most popular rule nodes by error count; + if (log.isTraceEnabled()) { + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); + } + if (pauseBetweenRetries > 0) { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(pauseBetweenRetries)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return new TbRuleEngineProcessingDecision(false, toReprocess); + } + } + } + } + + private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + + @Override + public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { + log.info("Skip reprocess of the rule engine pack"); + return new TbRuleEngineProcessingDecision(true, null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 816479b1e2..84fa1a7ac2 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,6 +32,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; +import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -502,7 +503,12 @@ public class DefaultDeviceStateService implements DeviceStateService { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state) , null, null, null); - actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new QueueToRuleEngineMsg(stateData.getTenantId(), tbMsg))); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, stateData.getTenantId(), stateData.getDeviceId()); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(stateData.getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + queueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index be186e0ddd..adff35fa51 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -187,7 +187,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onTimeseriesDataUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { + public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { onLocalSubUpdate(entityId, s -> { if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index d4d9fa5841..aa1824c855 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -31,7 +31,8 @@ public interface SubscriptionManagerService extends ApplicationListener ts, TbMsgCallback callback); + void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback); void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 58ed13c1e6..ab2803345a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -168,7 +168,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { - subscriptionManagerService.onTimeseriesDataUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8a4d735d26..c73e623d56 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -181,27 +181,27 @@ cassandra: # SQL configuration parameters sql: - # Specify batch size for persisting attribute updates - attributes: - batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" - ts: - batch_size: "${SQL_TS_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" - ts_latest: - batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" - # Specify whether to remove null characters from strValue of attributes and timeseries before insert - remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" - postgres: - # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. - ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" - timescale: - # Specify Interval size for new data chunks storage. - chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" + # Specify batch size for persisting attribute updates + attributes: + batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" + ts: + batch_size: "${SQL_TS_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" + ts_latest: + batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" + batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" + stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" + # Specify whether to remove null characters from strValue of attributes and timeseries before insert + remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" + postgres: + # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. + ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" + timescale: + # Specify Interval size for new data chunks storage. + chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" # Actor system parameters actors: @@ -330,19 +330,19 @@ updates: # spring CORS configuration spring.mvc.cors: - mappings: - # Intercept path - "[/api/**]": - #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. - allowed-origins: "*" - #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" - #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" - #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" - #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + mappings: + # Intercept path + "[/api/**]": + #Comma-separated list of origins to allow. '*' allows all origins. When not set,CORS support is disabled. + allowed-origins: "*" + #Comma-separated list of methods to allow. '*' allows all methods. + allowed-methods: "*" + #Comma-separated list of headers to allow in a request. '*' allows all headers. + allowed-headers: "*" + #How long, in seconds, the response from a pre-flight request can be cached by clients. + max-age: "1800" + #Set whether credentials are supported. When not set, credentials are not supported. + allow-credentials: "true" # spring serve gzip compressed static resources spring.resources.chain: @@ -551,6 +551,12 @@ queue: poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure_percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause_between_retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 9b43169737..062eb02749 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import lombok.AllArgsConstructor; import lombok.Builder; @@ -73,7 +74,10 @@ public final class TbMsg implements Serializable { log.warn("[{}] Created message with empty callback: {}", originator, type); this.callback = TbMsgCallback.EMPTY; } + } + public static ByteString toByteString(TbMsg msg) { + return ByteString.copyFrom(toByteArray(msg)); } public static byte[] toByteArray(TbMsg msg) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 1a80aa4a6e..42d3a534a9 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -244,7 +244,7 @@ public class DefaultTransportService implements TransportService { metaData.putValue("ts", tsKv.getTs() + ""); JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), - deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); + deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); sendToRuleEngine(tenantId, tbMsg, packCallback); } } @@ -261,7 +261,7 @@ public class DefaultTransportService implements TransportService { metaData.putValue("deviceName", sessionInfo.getDeviceName()); metaData.putValue("deviceType", sessionInfo.getDeviceType()); TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, - TbMsgDataType.JSON, gson.toJson(json), null, null, null); + TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); } } @@ -488,7 +488,7 @@ public class DefaultTransportService implements TransportService { protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(ByteString.copyFrom(TbMsg.toByteArray(tbMsg))) + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); From 7531a26e617092c437b9df6c8d13b22511eac1ab Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 Mar 2020 22:11:47 +0200 Subject: [PATCH 026/196] Implementation of RPC Call support --- .../server/actors/ActorSystemContext.java | 49 ++-- .../server/actors/app/AppActor.java | 6 +- .../server/actors/device/DeviceActor.java | 6 + .../device/DeviceActorMessageProcessor.java | 21 +- .../actors/ruleChain/DefaultTbContext.java | 46 +--- .../actors/ruleChain/RuleChainActor.java | 4 +- .../RuleChainActorMessageProcessor.java | 16 +- .../RuleNodeActorMessageProcessor.java | 6 +- .../server/actors/service/ActorService.java | 17 +- .../server/actors/service/ComponentActor.java | 6 +- .../actors/service/DefaultActorService.java | 81 +------ .../actors/shared/ComponentMsgProcessor.java | 11 +- .../server/actors/tenant/TenantActor.java | 28 ++- .../server/controller/BaseController.java | 21 +- .../server/controller/DeviceController.java | 15 +- .../server/controller/RpcController.java | 12 +- .../controller/RuleChainController.java | 12 +- .../controller/TelemetryController.java | 11 +- .../server/controller/TenantController.java | 2 +- .../encoding/DataDecodingEncodingService.java | 5 - .../service/encoding/ProtoWithFSTService.java | 24 +- .../queue/DefaultTbClusterService.java | 125 ++++++++++ .../queue/DefaultTbCoreConsumerService.java | 156 ++++++++++--- .../DefaultTbRuleEngineConsumerService.java | 103 ++++++-- .../server/service/queue/MsgPackCallback.java | 2 +- .../service/queue/TbClusterService.java | 37 +++ .../TbRuleEngineProcessingDecision.java | 15 ++ .../TbRuleEngineProcessingResult.java | 15 ++ .../TbRuleEngineProcessingStrategy.java | 15 ++ ...TbRuleEngineProcessingStrategyFactory.java | 15 ++ .../service/rpc/DefaultDeviceRpcService.java | 219 ------------------ .../rpc/DefaultTbCoreDeviceRpcService.java | 203 ++++++++++++++++ .../rpc/DefaultTbRuleEngineRpcService.java | 169 ++++++++++++++ .../server/service/rpc/DeviceRpcService.java | 41 ---- .../service/rpc/TbCoreDeviceRpcService.java | 57 +++++ .../rpc/TbRuleEngineDeviceRpcService.java | 16 +- .../rpc/ToDeviceRpcRequestActorMsg.java | 2 +- .../state/DefaultDeviceStateService.java | 58 ++--- .../DefaultSubscriptionManagerService.java | 37 ++- ...=> DefaultTbLocalSubscriptionService.java} | 12 +- ...e.java => TbLocalSubscriptionService.java} | 2 +- .../DefaultTelemetrySubscriptionService.java | 10 +- .../DefaultTelemetryWebSocketService.java | 4 +- .../DefaultTbCoreToTransportService.java | 8 +- application/src/main/proto/cluster.proto | 74 ------ .../src/main/resources/thingsboard.yml | 5 + ...AbstractRuleEngineFlowIntegrationTest.java | 6 +- ...actRuleEngineLifecycleIntegrationTest.java | 3 +- .../ConsistentHashParitionServiceTest.java | 4 +- .../server/common/msg/MsgType.java | 5 +- .../thingsboard/server/common/msg/TbMsg.java | 2 +- .../msg/plugin/ComponentLifecycleMsg.java | 2 +- .../PartitionChangeMsg.java} | 15 +- .../server/common/msg/queue}/ServiceKey.java | 2 +- .../server/common/msg/queue}/ServiceType.java | 2 +- .../common/msg/queue}/TopicPartitionInfo.java | 2 +- .../server/queue/TbQueueConsumer.java | 2 +- .../server/queue/TbQueueProducer.java | 2 +- .../common/DefaultTbQueueRequestTemplate.java | 2 +- .../DefaultTbQueueResponseTemplate.java | 2 +- .../discovery/ClusterTopologyChangeEvent.java | 1 + .../ConsistentHashPartitionService.java | 48 ++-- .../DefaultTbServiceInfoProvider.java | 1 + .../queue/discovery/PartitionChangeEvent.java | 2 + .../queue/discovery/PartitionService.java | 13 +- .../discovery/TbServiceInfoProvider.java | 1 + .../discovery/TopicPartitionInfoKey.java | 1 + .../queue/kafka/TBKafkaConsumerTemplate.java | 2 +- .../queue/kafka/TBKafkaProducerTemplate.java | 2 +- .../queue/memory/InMemoryTbQueueConsumer.java | 2 +- .../queue/memory/InMemoryTbQueueProducer.java | 2 +- .../InMemoryMonolithQueueProvider.java | 32 ++- ... => InMemoryTbTransportQueueProvider.java} | 10 +- .../provider/KafkaMonolithQueueProvider.java | 68 ++++-- .../provider/KafkaTbCoreQueueProvider.java | 77 ++++-- .../KafkaTbRuleEngineQueueProvider.java | 60 ++++- ...ava => KafkaTbTransportQueueProvider.java} | 14 +- .../provider/TbCoreQueueProducerProvider.java | 74 ++++++ .../queue/provider/TbCoreQueueProvider.java | 22 ++ .../provider/TbQueueProducerProvider.java | 66 ++++++ .../TbRuleEngineProducerProvider.java | 75 ++++++ .../provider/TbRuleEngineQueueProvider.java | 23 ++ .../TbTransportQueueProducerProvider.java | 70 ++++++ ...der.java => TbTransportQueueProvider.java} | 2 +- common/queue/src/main/proto/queue.proto | 20 +- .../transport/coap/CoapTransportService.java | 6 - .../service/DefaultTransportService.java | 16 +- .../api/RuleEngineDeviceRpcRequest.java | 5 +- .../rule/engine/api/RuleEngineRpcService.java | 4 +- .../thingsboard/rule/engine/api/TbNode.java | 4 +- .../rule/engine/debug/TbMsgGeneratorNode.java | 4 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 2 +- .../rule/engine/rpc/TbSendRPCRequestNode.java | 15 +- 93 files changed, 1720 insertions(+), 874 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java rename common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java => application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java (59%) rename application/src/main/java/org/thingsboard/server/service/subscription/{DefaultLocalSubscriptionService.java => DefaultTbLocalSubscriptionService.java} (96%) rename application/src/main/java/org/thingsboard/server/service/subscription/{LocalSubscriptionService.java => TbLocalSubscriptionService.java} (96%) delete mode 100644 application/src/main/proto/cluster.proto rename common/message/src/main/java/org/thingsboard/server/common/msg/{cluster/ClusterEventMsg.java => queue/PartitionChangeMsg.java} (72%) rename common/{queue/src/main/java/org/thingsboard/server/queue/discovery => message/src/main/java/org/thingsboard/server/common/msg/queue}/ServiceKey.java (96%) rename common/{queue/src/main/java/org/thingsboard/server/queue/discovery => message/src/main/java/org/thingsboard/server/common/msg/queue}/ServiceType.java (94%) rename common/{queue/src/main/java/org/thingsboard/server/queue/discovery => message/src/main/java/org/thingsboard/server/common/msg/queue}/TopicPartitionInfo.java (97%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{InMemoryTransportQueueProvider.java => InMemoryTbTransportQueueProvider.java} (90%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTransportQueueProvider.java => KafkaTbTransportQueueProvider.java} (91%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TransportQueueProvider.java => TbTransportQueueProvider.java} (97%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 64a713b336..03b65e7ccc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -45,6 +45,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; @@ -63,14 +65,9 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; @@ -78,7 +75,9 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.mail.MailExecutorService; -import org.thingsboard.server.service.rpc.DeviceRpcService; +import org.thingsboard.server.service.queue.TbClusterService; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.script.JsExecutorService; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.session.DeviceSessionCacheService; @@ -163,7 +162,11 @@ public class ActorSystemContext { @Autowired @Getter - private TbRuleEngineQueueProvider ruleEngineQueueProvider; + private TbClusterService clusterService; + + @Autowired + @Getter + private TbQueueProducerProvider producerProvider; @Autowired @Getter @@ -197,10 +200,6 @@ public class ActorSystemContext { @Getter private TelemetrySubscriptionService tsSubService; - @Autowired - @Getter - private DeviceRpcService deviceRpcService; - @Autowired @Getter private JsInvokeService jsSandbox; @@ -246,6 +245,22 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + /** + * The following Service will be null if we operate in tb-core mode + */ + @Lazy + @Getter + @Autowired(required = false) + private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; + + /** + * The following Service will be null if we operate in tb-rule-engine mode + */ + @Lazy + @Getter + @Autowired(required = false) + private TbCoreDeviceRpcService tbCoreDeviceRpcService; + @Lazy @Autowired @Getter @@ -406,14 +421,6 @@ public class ActorSystemContext { return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); } - public TbQueueProducer> getTbCoreMsgProducer() { - return ruleEngineQueueProvider.getTbCoreMsgProducer(); - } - - public TbQueueProducer> getRuleEngineMsgProducer() { - return ruleEngineQueueProvider.getRuleEngineMsgProducer(); - } - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { return partitionService.resolve(serviceType, tenantId, entityId); } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 9d80d63b2a..c096238a8b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -39,7 +39,9 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; @@ -81,7 +83,7 @@ public class AppActor extends RuleChainManagerActor { case SEND_TO_CLUSTER_MSG: onPossibleClusterMsg((SendToClusterMsg) msg); break; - case CLUSTER_EVENT_MSG: + case PARTITION_CHANGE_MSG: broadcast(msg); break; case COMPONENT_LIFE_CYCLE_MSG: @@ -131,7 +133,7 @@ public class AppActor extends RuleChainManagerActor { // systemContext.getRpcService().tell( // systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); // } else { - self().tell(msg.getMsg(), ActorRef.noSender()); + self().tell(msg.getMsg(), ActorRef.noSender()); // } } diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 547e484652..3a15240f33 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; @@ -48,6 +49,11 @@ public class DeviceActor extends ContextAwareActor { } } + @Override + public void postStop() { + + } + @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 4713662067..aff6200199 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -16,13 +16,10 @@ package org.thingsboard.server.actors.device; import akka.actor.ActorContext; -import com.datastax.driver.core.utils.UUIDs; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.google.gson.JsonObject; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -38,11 +35,8 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.gen.transport.TransportProtos; @@ -52,8 +46,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestM import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; -import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; @@ -63,14 +55,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; -import org.thingsboard.server.common.transport.util.JsonUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -103,8 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map toDeviceRpcPendingMap; private final Map toServerRpcPendingMap; - private final Gson gson = new Gson(); - private int rpcSeq = 0; private String deviceName; private String deviceType; @@ -162,7 +150,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (request.isOneway() && sent) { log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null)); } else { registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout); } @@ -183,7 +171,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId()); if (requestMd != null) { log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION)); } } @@ -215,7 +203,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestBody body = request.getBody(); if (request.isOneway()) { sentOneWayIds.add(entry.getKey()); - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null)); } ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId( entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build(); @@ -417,7 +405,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId()); boolean success = requestMd != null; if (success) { - systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), + systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(), responseMsg.getPayload(), null)); } else { log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId()); @@ -689,4 +677,5 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { dumpSessions(); } } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index c9273e0a11..7c831929ca 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -48,9 +48,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -67,8 +65,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -155,12 +153,7 @@ class DefaultTbContext implements TbContext { @Override public void sendTbMsgToRuleEngine(TbMsg tbMsg) { - TenantId tenantId = getTenantId(); - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); - mainCtx.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + mainCtx.getClusterService().onToRuleEngineMsg(getTenantId(), tbMsg.getOriginator(), tbMsg); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { @@ -351,36 +344,7 @@ class DefaultTbContext implements TbContext { @Override public RuleEngineRpcService getRpcService() { - return new RuleEngineRpcService() { - @Override - public void sendRpcReply(DeviceId deviceId, int requestId, String body) { - mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body); - } - - @Override - public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer consumer) { - ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(), - src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); - mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> { - if (src.isRestApiCall()) { - //TODO 2.5 -// ServerAddress requestOriginAddress; -// if (!StringUtils.isEmpty(src.getOriginHost())) { -// requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE); -// } else { -// requestOriginAddress = mainCtx.getRoutingService().getCurrentServer(); -// } -// mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response); - } - consumer.accept(RuleEngineDeviceRpcResponse.builder() - .deviceId(src.getDeviceId()) - .requestId(src.getRequestId()) - .error(response.getError()) - .response(response.getResponse()) - .build()); - }); - } - }; + return mainCtx.getTbRuleEngineDeviceRpcService(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 83cde28a45..1b9923139a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import scala.concurrent.duration.Duration; @@ -51,7 +52,8 @@ public class RuleChainActor extends ComponentActor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.producer = systemContext.getRuleEngineQueueProvider().getRuleEngineMsgProducer(); + this.producer = systemContext.getProducerProvider().getRuleEngineMsgProducer(); } @Override @@ -153,8 +151,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actorRef.tell(msg, self)); } private ActorRef createRuleNodeActor(ActorContext context, RuleNode ruleNode) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index b4bda98f7a..2dd6764407 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.dao.rule.RuleChainService; /** @@ -84,9 +84,9 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor status = system.terminate(); @@ -108,68 +107,10 @@ public class DefaultActorService implements ActorService { } } - @Override - public void onMsg(TbActorMsg msg) { - appActor.tell(msg, ActorRef.noSender()); - } - - //TODO 2.5 -// @Override -// public void onServerAdded(ServerInstance server) { -// log.trace("Processing onServerAdded msg: {}", server); -// broadcast(new ClusterEventMsg(server.getServerAddress(), true)); -// } -// -// @Override -// public void onServerUpdated(ServerInstance server) { -// //Do nothing -// } -// -// @Override -// public void onServerRemoved(ServerInstance server) { -// log.trace("Processing onServerRemoved msg: {}", server); -// broadcast(new ClusterEventMsg(server.getServerAddress(), false)); -// } - - @Override - public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { - log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); - broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); - } - - @Override - public void onCredentialsUpdate(TenantId tenantId, DeviceId deviceId) { - DeviceCredentialsUpdateNotificationMsg msg = new DeviceCredentialsUpdateNotificationMsg(tenantId, deviceId); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - @Override - public void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType) { - log.trace("[{}] Processing onDeviceNameOrTypeUpdate event, deviceName: {}, deviceType: {}", deviceId, deviceName, deviceType); - DeviceNameOrTypeUpdateMsg msg = new DeviceNameOrTypeUpdateMsg(tenantId, deviceId, deviceName, deviceType); - appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender()); - } - - public void broadcast(ToAllNodesMsg msg) { - actorContext.getEncodingService().encode(msg); - //TODO 2.5 -// rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage -// .newBuilder() -// .setPayload(ByteString -// .copyFrom(actorContext.getEncodingService().encode(msg))) -// .setMessageType(CLUSTER_ACTOR_MESSAGE) -// .build())); - appActor.tell(msg, ActorRef.noSender()); - } - - private void broadcast(ClusterEventMsg msg) { - this.appActor.tell(msg, ActorRef.noSender()); - this.rpcManagerActor.tell(msg, ActorRef.noSender()); - } - @Value("${cluster.stats.enabled:false}") private boolean statsEnabled; + //TODO 2.5 private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); @@ -258,8 +199,4 @@ public class DefaultActorService implements ActorService { // rpcManagerActor.tell(msg, ActorRef.noSender()); // } - @Override - public void onDeviceAdded(Device device) { - deviceStateService.onDeviceAdded(device); - } } diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java index 2fdc918f8e..1443c61e16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/shared/ComponentMsgProcessor.java @@ -16,20 +16,13 @@ package org.thingsboard.server.actors.shared; import akka.actor.ActorContext; -import akka.event.LoggingAdapter; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.stats.StatsPersistTick; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; - -import javax.annotation.Nullable; -import java.util.function.Consumer; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @Slf4j public abstract class ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -50,7 +43,7 @@ public abstract class ComponentMsgProcessor extends Abstract public abstract void stop(ActorContext context) throws Exception; - public abstract void onClusterEventMsg(ClusterEventMsg msg) throws Exception; + public abstract void onPartitionChangeMsg(PartitionChangeMsg msg) throws Exception; public void onCreated(ActorContext context) throws Exception { start(context); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index d47591f252..80629e5927 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -41,9 +41,14 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; +import java.util.List; +import java.util.stream.Collectors; + public class TenantActor extends RuleChainManagerActor { private final TenantId tenantId; @@ -79,8 +84,21 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { switch (msg.getMsgType()) { - case CLUSTER_EVENT_MSG: - broadcast(msg); + case PARTITION_CHANGE_MSG: + PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; + ServiceType serviceType = partitionChangeMsg.getServiceKey().getServiceType(); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + //To Rule Chain Actors + broadcast(msg); + } else if (ServiceType.TB_CORE.equals(serviceType)) { + //To Device Actors + List repartitionedDevices = + deviceActors.keySet().stream().filter(deviceId -> !isMyPartition(deviceId)).collect(Collectors.toList()); + for (DeviceId deviceId : repartitionedDevices) { + ActorRef deviceActor = deviceActors.remove(deviceId); + context().stop(deviceActor); + } + } break; case COMPONENT_LIFE_CYCLE_MSG: onComponentLifecycleMsg((ComponentLifecycleMsg) msg); @@ -106,6 +124,10 @@ public class TenantActor extends RuleChainManagerActor { return true; } + private boolean isMyPartition(DeviceId deviceId) { + return systemContext.resolve(ServiceType.TB_CORE, tenantId, deviceId).isMyPartition(); + } + private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { TbMsg tbMsg = msg.getTbMsg(); if (tbMsg.getRuleChainId() == null) { @@ -169,7 +191,7 @@ public class TenantActor extends RuleChainManagerActor { if (removed) { log.debug("[{}] Removed actor:", terminated); } else { - log.warn("Removed actor was not found in the device map!"); + log.debug("Removed actor was not found in the device map!"); } } else { throw new IllegalStateException("Remote actors are not supported!"); diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index bc88544243..9c8ba94ccf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -27,7 +27,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; @@ -71,8 +70,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -93,13 +90,10 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; @@ -168,7 +162,7 @@ public abstract class BaseController { protected RuleChainService ruleChainService; @Autowired - protected ActorService actorService; + protected TbClusterService tbClusterService; @Autowired protected RelationService relationService; @@ -195,7 +189,7 @@ public abstract class BaseController { protected PartitionService partitionService; @Autowired - protected TbCoreQueueProvider coreQueueProvider; + protected TbQueueProducerProvider producerProvider; @Value("${server.log_controller_error_stack_trace}") @Getter @@ -674,12 +668,7 @@ public abstract class BaseController { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON , json.writeValueAsString(entityNode) , null, null, null); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, user.getTenantId(), entityId); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(user.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(user.getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - coreQueueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + tbClusterService.onToRuleEngineMsg(user.getTenantId(), entityId, tbMsg); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index d147d27b40..562d46d9f0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -30,6 +30,8 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -94,12 +96,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - actorService - .onDeviceNameOrTypeUpdate( - savedDevice.getTenantId(), - savedDevice.getId(), - savedDevice.getName(), - savedDevice.getType()); + tbClusterService.onToCoreMsg(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType())); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -252,7 +250,9 @@ public class DeviceController extends BaseController { try { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - actorService.onCredentialsUpdate(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()); + + tbClusterService.onToCoreMsg(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId())); + logEntityAction(device.getId(), device, device.getCustomerId(), ActionType.CREDENTIALS_UPDATED, null, deviceCredentials); @@ -425,6 +425,7 @@ public class DeviceController extends BaseController { deferredResult.setResult(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); } } + @Override public void onFailure(Throwable t) { deferredResult.setErrorResult(t); diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index a8298ce82b..2d747f32e6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.FutureCallback; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -31,7 +32,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -43,22 +43,18 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.service.rpc.DeviceRpcService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.telemetry.exception.ToErrorResponseEntity; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Created by ashvayka on 22.03.18. @@ -72,7 +68,7 @@ public class RpcController extends BaseController { protected final ObjectMapper jsonMapper = new ObjectMapper(); @Autowired - private DeviceRpcService deviceRpcService; + private TbCoreDeviceRpcService deviceRpcService; @Autowired private AccessValidator accessValidator; @@ -116,7 +112,7 @@ public class RpcController extends BaseController { timeout, body ); - deviceRpcService.processRestAPIRpcRequestToRuleEngine(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); + deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 3115136713..9dc1974e2a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -131,7 +131,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - actorService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), savedRuleChain.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, @@ -162,7 +162,7 @@ public class RuleChainController extends BaseController { previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - actorService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, @@ -170,7 +170,7 @@ public class RuleChainController extends BaseController { ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, @@ -204,7 +204,7 @@ public class RuleChainController extends BaseController { RuleChain ruleChain = checkRuleChain(ruleChainMetaData.getRuleChainId(), Operation.WRITE); RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.UPDATED); logEntityAction(ruleChain.getId(), ruleChain, null, @@ -255,9 +255,9 @@ public class RuleChainController extends BaseController { referencingRuleChainIds.remove(ruleChain.getId()); referencingRuleChainIds.forEach(referencingRuleChainId -> - actorService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); - actorService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), ComponentLifecycleEvent.DELETED); logEntityAction(ruleChainId, ruleChain, null, diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index e534ebf7c4..b2b3809dd2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -68,7 +68,6 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.service.security.AccessValidator; @@ -362,9 +361,8 @@ public class TelemetryController extends BaseController { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify)); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -398,9 +396,8 @@ public class TelemetryController extends BaseController { logAttributesUpdated(user, entityId, scope, attributes, null); if (entityId.getEntityType() == EntityType.DEVICE) { DeviceId deviceId = new DeviceId(entityId.getId()); - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes); - actorService.onMsg(new SendToClusterMsg(deviceId, notificationMsg)); + tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes)); } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 8d95e78cfa..a89658ba0c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -95,7 +95,7 @@ public class TenantController extends BaseController { checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); + tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java index a5d3ab465e..4a781b8673 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/DataDecodingEncodingService.java @@ -16,8 +16,6 @@ package org.thingsboard.server.service.encoding; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; @@ -27,8 +25,5 @@ public interface DataDecodingEncodingService { byte[] encode(TbActorMsg msq); - ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg); - } diff --git a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java index 45bb9f78fd..8d89059488 100644 --- a/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java +++ b/application/src/main/java/org/thingsboard/server/service/encoding/ProtoWithFSTService.java @@ -15,25 +15,19 @@ */ package org.thingsboard.server.service.encoding; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; import java.util.Optional; -import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE; - - @Slf4j @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + @Override public Optional decode(byte[] byteArray) { try { @@ -42,7 +36,7 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); - return Optional.empty(); + return Optional.empty(); } } @@ -51,18 +45,4 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { return config.asByteArray(msq); } - @Override - public ClusterAPIProtos.ClusterMessage convertToProtoDataMessage(ServerAddress serverAddress, - TbActorMsg msg) { - return ClusterAPIProtos.ClusterMessage - .newBuilder() - .setServerAddress(ClusterAPIProtos.ServerAddress - .newBuilder() - .setHost(serverAddress.getHost()) - .setPort(serverAddress.getPort()) - .build()) - .setMessageType(CLUSTER_ACTOR_MESSAGE) - .setPayload(ByteString.copyFrom(encode(msg))).build(); - - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java new file mode 100644 index 0000000000..5851fd919c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +import java.util.HashSet; +import java.util.Set; + +@Service +@Slf4j +public class DefaultTbClusterService implements TbClusterService { + + protected TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final DataDecodingEncodingService encodingService; + + public DefaultTbClusterService(TbQueueProducerProvider producerProvider, PartitionService partitionService, DataDecodingEncodingService encodingService) { + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.encodingService = encodingService; + } + + @Override + public void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + } + + @Override + public void onToCoreMsg(ToDeviceActorNotificationMsg msg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + byte[] msgBytes = encodingService.encode(msg); + ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), null); + } + + @Override + public void onToCoreMsg(String serviceId, FromDeviceRpcResponse response) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + } + + @Override + public void onToRuleEngineMsg(String serviceId, FromDeviceRpcResponse response) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() + .setRequestIdMSB(response.getId().getMostSignificantBits()) + .setRequestIdLSB(response.getId().getLeastSignificantBits()) + .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); + response.getResponse().ifPresent(builder::setResponse); + ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + + } + + @Override + public void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); + } + + private void broadcast(ComponentLifecycleMsg msg) { + byte[] msgBytes = encodingService.encode(msg); + TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { + TbQueueProducer> toCoreProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); + for (String serviceId : tbCoreServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toCoreProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + } + // No need to push notifications twice + tbRuleEngineServices.removeAll(tbCoreServices); + } + for (String serviceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 30148dfe08..4e2387082b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,26 +24,32 @@ import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; import org.thingsboard.server.service.state.DeviceStateService; -import org.thingsboard.server.service.subscription.LocalSubscriptionService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,64 +74,85 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { private final ActorSystemContext actorContext; private final DeviceStateService stateService; - private final LocalSubscriptionService localSubscriptionService; + private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; - private final TbQueueConsumer> consumer; + private final DataDecodingEncodingService encodingService; + private final TbQueueConsumer> mainConsumer; + private final TbQueueConsumer> nfConsumer; + private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private volatile ExecutorService mainConsumerExecutor; + private volatile ExecutorService notificationsConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService, LocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService) { - this.consumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, + DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, + SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, + TbCoreDeviceRpcService tbCoreDeviceRpcService) { + this.mainConsumer = tbCoreQueueProvider.getToCoreMsgConsumer(); + this.nfConsumer = tbCoreQueueProvider.getToCoreNotificationsMsgConsumer(); this.actorContext = actorContext; this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; + this.encodingService = encodingService; + this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; } @PostConstruct public void init() { this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-notifications-consumer")); } @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_CORE) { log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.consumer.subscribe(partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); } } @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", mainConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumer(); + } + + private void launchMainConsumer() { mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = mainConsumer.poll(pollDuration); if (msgs.isEmpty()) { continue; } ConcurrentMap> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> successMap = new ConcurrentHashMap<>(); ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, successMap, failedMap); + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToDeviceActorMsg()) { + if (toCoreMsg.hasToSubscriptionMgrMsg()) { + log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); + forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); + } else if (toCoreMsg.hasToDeviceActorMsg()) { log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg()); forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback); } else if (toCoreMsg.hasDeviceStateServiceMsg()) { log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg()); forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback); - } else if (toCoreMsg.hasToSubscriptionMgrMsg()) { - log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); - forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); - } else if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -136,7 +163,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } - consumer.commit(); + mainConsumer.commit(); } catch (Exception e) { log.warn("Failed to obtain messages from queue.", e); try { @@ -150,6 +177,67 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { }); } + private void launchNotificationsConsumer() { + notificationsConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + try { + ToCoreNotificationMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreMsg.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); + } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + log.info("Tb Core Notifications Consumer stopped."); + }); + } + + private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbMsgCallback callback) { + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) + , proto.getResponse(), error); + tbCoreDeviceRpcService.processRpcResponseFromRuleEngine(response); + callback.onSuccess(); + } + @Scheduled(fixedDelayString = "${queue.core.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { @@ -160,15 +248,21 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { @PreDestroy public void destroy() { stopped = true; - if (consumer != null) { - consumer.unsubscribe(); + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (nfConsumer != null) { + nfConsumer.unsubscribe(); } if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } } - private void forwardToLocalSubMgrService(TransportProtos.LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { + private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { if (msg.hasSubUpdate()) { localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); } else { @@ -176,22 +270,22 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } - private void forwardToSubMgrService(TransportProtos.SubscriptionMgrMsgProto msg, TbMsgCallback callback) { + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbMsgCallback callback) { if (msg.hasAttributeSub()) { subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getAttributeSub()), callback); } else if (msg.hasTelemetrySub()) { subscriptionManagerService.addSubscription(TbSubscriptionUtils.fromProto(msg.getTelemetrySub()), callback); } else if (msg.hasSubClose()) { - TransportProtos.TbSubscriptionCloseProto closeProto = msg.getSubClose(); + TbSubscriptionCloseProto closeProto = msg.getSubClose(); subscriptionManagerService.cancelSubscription(closeProto.getSessionId(), closeProto.getSubscriptionId(), callback); } else if (msg.hasTsUpdate()) { - TransportProtos.TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); + TbTimeSeriesUpdateProto proto = msg.getTsUpdate(); subscriptionManagerService.onTimeSeriesUpdate( new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), TbSubscriptionUtils.toTsKvEntityList(proto.getDataList()), callback); } else if (msg.hasAttrUpdate()) { - TransportProtos.TbAttributeUpdateProto proto = msg.getAttrUpdate(); + TbAttributeUpdateProto proto = msg.getAttrUpdate(); subscriptionManagerService.onAttributesUpdate( new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), TbSubscriptionUtils.toEntityId(proto.getEntityType(), proto.getEntityIdMSB(), proto.getEntityIdLSB()), @@ -201,7 +295,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } - private void forwardToStateService(TransportProtos.DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { + private void forwardToStateService(DeviceStateServiceMsgProto deviceStateServiceMsg, TbMsgCallback callback) { if (statsEnabled) { stats.log(deviceStateServiceMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 986d858043..b9d2b98d04 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,16 +26,19 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; @@ -44,6 +47,7 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -67,21 +71,28 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS private boolean statsEnabled; private final ActorSystemContext actorContext; - private final TbQueueConsumer> consumer; + private final TbQueueConsumer> mainConsumer; + private final TbQueueConsumer> nfConsumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private final TbRuleEngineProcessingStrategyFactory factory; + private final DataDecodingEncodingService encodingService; private volatile ExecutorService mainConsumerExecutor; + private volatile ExecutorService notificationsConsumerExecutor; private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext) { + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext, + DataDecodingEncodingService encodingService) { this.factory = factory; - this.consumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); + this.mainConsumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); + this.nfConsumer = tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer(); this.actorContext = actorContext; + this.encodingService = encodingService; } @PostConstruct public void init() { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-notifications-consumer")); this.factory.newInstance(); } @@ -89,16 +100,70 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_RULE_ENGINE) { log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.consumer.subscribe(partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); } } @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumer(); + } + + private void launchNotificationsConsumer() { + notificationsConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + try { + ToRuleEngineNotificationMsg toRuleEngineMsg = msg.getValue(); + if (toRuleEngineMsg.getComponentLifecycleMsg() != null && !toRuleEngineMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toRuleEngineMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else { + callback.onSuccess(); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + }); + } + + private void launchMainConsumer() { mainConsumerExecutor.execute(() -> { while (!stopped) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = mainConsumer.poll(pollDuration); if (msgs.isEmpty()) { continue; } @@ -106,7 +171,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS TbRuleEngineProcessingDecision decision = null; boolean firstAttempt = true; while (!stopped && (firstAttempt || !decision.isCommit())) { - ConcurrentMap> allMap; + ConcurrentMap> allMap; if (firstAttempt) { allMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); @@ -114,14 +179,14 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } else { allMap = decision.getReprocessMap(); } - ConcurrentMap> successMap = new ConcurrentHashMap<>(); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + ConcurrentMap> successMap = new ConcurrentHashMap<>(); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); allMap.forEach((id, msg) -> { TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); try { - TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); @@ -139,7 +204,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); } - consumer.commit(); + mainConsumer.commit(); } catch (Exception e) { log.warn("Failed to process messages from queue.", e); try { @@ -171,11 +236,17 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @PreDestroy public void destroy() { stopped = true; - if (consumer != null) { - consumer.unsubscribe(); + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (nfConsumer != null) { + nfConsumer.unsubscribe(); } if (mainConsumerExecutor != null) { mainConsumerExecutor.shutdownNow(); } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java index 420c61d5d7..72c5c10458 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java new file mode 100644 index 0000000000..952b88e4d3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; + +public interface TbClusterService { + + void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg msg); + + void onToCoreMsg(ToDeviceActorNotificationMsg msg); + + void onToCoreMsg(String targetServiceId, FromDeviceRpcResponse response); + + void onToRuleEngineMsg(String targetServiceId, FromDeviceRpcResponse response); + + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java index b116fd7443..4a4829f0c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingDecision.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index 601a3e9a9b..c17cf54bca 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java index c644766892..cfba85f5dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; public interface TbRuleEngineProcessingStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 336baf5360..8d45959df3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java deleted file mode 100644 index d8f12b8f0a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultDeviceRpcService.java +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.gen.cluster.ClusterAPIProtos; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * Created by ashvayka on 27.03.18. - */ -@Service -@Slf4j -public class DefaultDeviceRpcService implements DeviceRpcService { - - private static final ObjectMapper json = new ObjectMapper(); - - @Autowired - private DeviceService deviceService; - - @Autowired - @Lazy - private ActorService actorService; - - private ScheduledExecutorService rpcCallBackExecutor; - - private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); - private final ConcurrentMap> localToDeviceRpcRequests = new ConcurrentHashMap<>(); - - @PostConstruct - public void initExecutor() { - rpcCallBackExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rpc-callback")); - } - - @PreDestroy - public void shutdownExecutor() { - if (rpcCallBackExecutor != null) { - rpcCallBackExecutor.shutdownNow(); - } - } - - @Override - public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToRuleEngineRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToRuleEngine(request); - scheduleTimeout(request, requestId, localToRuleEngineRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress); - //TODO 2.5 - if (true) {//routingService.getCurrentServer().equals(requestOriginAddress) - UUID requestId = response.getId(); - Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } else { - ClusterAPIProtos.FromDeviceRPCResponseProto.Builder builder = ClusterAPIProtos.FromDeviceRPCResponseProto.newBuilder(); - builder.setRequestIdMSB(response.getId().getMostSignificantBits()); - builder.setRequestIdLSB(response.getId().getLeastSignificantBits()); - response.getResponse().ifPresent(builder::setResponse); - if (response.getError().isPresent()) { - builder.setError(response.getError().get().ordinal()); - } else { - builder.setError(-1); - } - //TODO 2.5 -// rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray()); - } - } - - @Override - public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { - log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); - UUID requestId = request.getId(); - localToDeviceRpcRequests.put(requestId, responseConsumer); - sendRpcRequestToDevice(request); - scheduleTimeout(request, requestId, localToDeviceRpcRequests); - } - - @Override - public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); - UUID requestId = response.getId(); - Consumer consumer = localToDeviceRpcRequests.remove(requestId); - if (consumer != null) { - consumer.accept(response); - } else { - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); - } - } - - @Override - public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) { - ClusterAPIProtos.FromDeviceRPCResponseProto proto; - try { - proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error); - //TODO 2.5 -// processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response); - } - - @Override - public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) { - ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body)); - forward(deviceId, rpcMsg); - } - - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { - ObjectNode entityNode = json.createObjectNode(); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("requestUUID", msg.getId().toString()); - //TODO 2.5 -// metaData.putValue("originHost", routingService.getCurrentServer().getHost()); -// metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort())); - metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(msg.isOneway())); - - Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); - if (device != null) { - metaData.putValue("deviceName", device.getName()); - metaData.putValue("deviceType", device.getType()); - } - - entityNode.put("method", msg.getBody().getMethod()); - entityNode.put("params", msg.getBody().getParams()); - - try { - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON - , json.writeValueAsString(entityNode) - , null, null, null); - actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new QueueToRuleEngineMsg(msg.getTenantId(), tbMsg))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { - //TODO 2.5 -// ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(routingService.getCurrentServer(), msg); -// log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); -// forward(msg.getDeviceId(), rpcMsg); - } - - private void forward(DeviceId deviceId, T msg) { - actorService.onMsg(new SendToClusterMsg(deviceId, msg)); - } - - private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId, ConcurrentMap> requestsMap) { - long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); - rpcCallBackExecutor.schedule(() -> { - log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); - Consumer consumer = requestsMap.remove(requestId); - if (consumer != null) { - consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); - } - }, timeout, TimeUnit.MILLISECONDS); - } - - -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java new file mode 100644 index 0000000000..ed55ee11df --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -0,0 +1,203 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import akka.actor.ActorRef; +import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Created by ashvayka on 27.03.18. + */ +@Service +@Slf4j +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { + + private static final ObjectMapper json = new ObjectMapper(); + + private final DeviceService deviceService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + private final ActorSystemContext actorContext; + + private final ConcurrentMap> localToRuleEngineRpcRequests = new ConcurrentHashMap<>(); + private final ConcurrentMap localToDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbRuleEngineRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbCoreDeviceRpcService(DeviceService deviceService, TbClusterService clusterService, TbServiceInfoProvider serviceInfoProvider, + ActorSystemContext actorContext) { + this.deviceService = deviceService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + this.actorContext = actorContext; + } + + @Autowired + public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { + this.tbRuleEngineRpcService = tbRuleEngineRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-core-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToRuleEngineRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToRuleEngine(request); + scheduleToRuleEngineTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + @Override + public void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg rpcMsg) { + ToDeviceRpcRequest request = rpcMsg.getMsg(); + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + localToDeviceRpcRequests.put(requestId, rpcMsg); + actorContext.getAppActor().tell(rpcMsg, ActorRef.noSender()); + scheduleToDeviceTimeout(request, requestId); + } + + @Override + public void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId()); + UUID requestId = response.getId(); + ToDeviceRpcRequestActorMsg request = localToDeviceRpcRequests.remove(requestId); + if (request != null) { + sendRpcResponseToTbRuleEngine(request.getServiceId(), response); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void sendRpcResponseToTbRuleEngine(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbRuleEngineRpcService.isPresent()) { + tbRuleEngineRpcService.get().processRpcResponseFromDevice(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.onToRuleEngineMsg(originServiceId, response); + } + } + + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) { + ObjectNode entityNode = json.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("requestUUID", msg.getId().toString()); + metaData.putValue("originServiceId", serviceId); + metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(msg.isOneway())); + + Device device = deviceService.findDeviceById(msg.getTenantId(), msg.getDeviceId()); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + } + + entityNode.put("method", msg.getBody().getMethod()); + entityNode.put("params", msg.getBody().getParams()); + + try { + TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON + , json.writeValueAsString(entityNode) + , null, null, null); + clusterService.onToRuleEngineMsg(msg.getTenantId(), msg.getDeviceId(), tbMsg); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId); + Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); + if (consumer != null) { + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); + } + }, timeout, TimeUnit.MILLISECONDS); + } + + private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId); + localToDeviceRpcRequests.remove(requestId); + }, timeout, TimeUnit.MILLISECONDS); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java new file mode 100644 index 0000000000..6481f1c2b6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.RpcError; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; +import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.service.queue.TbClusterService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +@Slf4j +public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { + + private final PartitionService partitionService; + private final TbClusterService clusterService; + private final TbServiceInfoProvider serviceInfoProvider; + + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); + + private Optional tbCoreRpcService; + private ScheduledExecutorService scheduler; + private String serviceId; + + public DefaultTbRuleEngineRpcService(PartitionService partitionService, + TbClusterService clusterService, + TbServiceInfoProvider serviceInfoProvider) { + this.partitionService = partitionService; + this.clusterService = clusterService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Autowired + public void setTbCoreRpcService(Optional tbCoreRpcService) { + this.tbCoreRpcService = tbCoreRpcService; + } + + @PostConstruct + public void initExecutor() { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("rule-engine-rpc-scheduler")); + serviceId = serviceInfoProvider.getServiceId(); + } + + @PreDestroy + public void shutdownExecutor() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body) { +// TODO 2.5 + } + + @Override + public void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest src, Consumer consumer) { + ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), src.getTenantId(), src.getDeviceId(), + src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody())); + forwardRpcRequestToDeviceActor(request, response -> { + if (src.isRestApiCall()) { + sendRpcResponseToTbCore(src.getOriginServiceId(), response); + } + consumer.accept(RuleEngineDeviceRpcResponse.builder() + .deviceId(src.getDeviceId()) + .requestId(src.getRequestId()) + .error(response.getError()) + .response(response.getResponse()) + .build()); + }); + } + + @Override + public void processRpcResponseFromDevice(FromDeviceRpcResponse response) { + log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId()); + UUID requestId = response.getId(); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(response)); + } else { + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response); + } + } + + private void forwardRpcRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer) { + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); + UUID requestId = request.getId(); + toDeviceRpcRequests.put(requestId, responseConsumer); + sendRpcRequestToDevice(request); + scheduleTimeout(request, requestId); + } + + private void sendRpcRequestToDevice(ToDeviceRpcRequest msg) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + ToDeviceRpcRequestActorMsg rpcMsg = new ToDeviceRpcRequestActorMsg(serviceId, msg); + if (tpi.isMyPartition()) { + log.trace("[{}] Forwarding msg {} to device actor!", msg.getDeviceId(), msg); + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().forwardRpcRequestToDeviceActor(rpcMsg); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); + clusterService.onToCoreMsg(rpcMsg); + } + } + + private void sendRpcResponseToTbCore(String originServiceId, FromDeviceRpcResponse response) { + if (serviceId.equals(originServiceId)) { + if (tbCoreRpcService.isPresent()) { + tbCoreRpcService.get().processRpcResponseFromRuleEngine(response); + } else { + log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); + } + } else { + clusterService.onToCoreMsg(originServiceId, response); + } + } + + private void scheduleTimeout(ToDeviceRpcRequest request, UUID requestId) { + long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); + log.trace("[{}] processing the request: [{}]", this.hashCode(), requestId); + scheduler.schedule(() -> { + log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId); + Consumer consumer = toDeviceRpcRequests.remove(requestId); + if (consumer != null) { + scheduler.submit(() -> consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT))); + } + }, timeout, TimeUnit.MILLISECONDS); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java deleted file mode 100644 index feb5d58c3d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DeviceRpcService.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; - -import java.util.function.Consumer; - -/** - * Created by ashvayka on 16.04.18. - */ -public interface DeviceRpcService { - - void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response); - - void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer responseConsumer); - - void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response); - - void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data); - - void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body); -} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java new file mode 100644 index 0000000000..896745c48e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbCoreDeviceRpcService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.rpc; + +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; + +import java.util.function.Consumer; + +/** + * Handles REST API calls that contain RPC requests to Device. + */ +public interface TbCoreDeviceRpcService { + + /** + * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine. + * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest} + * + * @param request the RPC request + * @param responseConsumer the consumer of the RPC response + */ + void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer responseConsumer); + + /** + * Handles the RPC response from the Rule Engine. + * + * @param response the RPC response + */ + void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response); + + /** + * Forwards the RPC request from Rule Engine to Device Actor + * + * @param request the RPC request message + */ + void forwardRpcRequestToDeviceActor(ToDeviceRpcRequestActorMsg request); + + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDeviceActor(FromDeviceRpcResponse response); + +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java similarity index 59% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java rename to application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java index ed1bbf18b5..3d4f23a796 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/TbRuleEngineDeviceRpcService.java @@ -13,12 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport; +package org.thingsboard.server.service.rpc; -import org.thingsboard.server.common.data.Device; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; -public interface SessionMsgProcessor { +/** + * Created by ashvayka on 16.04.18. + */ +public interface TbRuleEngineDeviceRpcService extends RuleEngineRpcService { - void onDeviceAdded(Device device); + /** + * Handles the RPC response from the Device Actor (Transport). + * + * @param response the RPC response + */ + void processRpcResponseFromDevice(FromDeviceRpcResponse response); } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index ba537aed25..3a1ea599e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -35,7 +35,7 @@ import java.util.Optional; public class ToDeviceRpcRequestActorMsg implements ToDeviceActorNotificationMsg { @Getter - private final ServerAddress serverAddress; + private final String serviceId; @Getter private final ToDeviceRpcRequest msg; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 84fa1a7ac2..91a49aae1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +32,6 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.service.ActorService; -import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -50,19 +49,17 @@ import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -106,30 +103,13 @@ public class DefaultDeviceStateService implements DeviceStateService { public static final List PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); - @Autowired - private TenantService tenantService; - - @Autowired - private DeviceService deviceService; - - @Autowired - private AttributesService attributesService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - @Lazy - private ActorService actorService; - - @Autowired - private TbCoreQueueProvider queueProvider; - - @Autowired - private PartitionService partitionService; - - @Autowired - private TelemetrySubscriptionService tsSubService; + private final TenantService tenantService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService tsService; + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private final TelemetrySubscriptionService tsSubService; @Value("${state.defaultInactivityTimeoutInSec}") @Getter @@ -155,6 +135,18 @@ public class DefaultDeviceStateService implements DeviceStateService { private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, + AttributesService attributesService, TimeseriesService tsService, + TbQueueProducerProvider producerProvider, PartitionService partitionService, TelemetrySubscriptionService tsSubService) { + this.tenantService = tenantService; + this.deviceService = deviceService; + this.attributesService = attributesService; + this.tsService = tsService; + this.producerProvider = producerProvider; + this.partitionService = partitionService; + this.tsSubService = tsSubService; + } + @PostConstruct public void init() { // Should be always single threaded due to absence of locks. @@ -429,7 +421,7 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setUpdated(updated); builder.setDeleted(deleted); TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); - queueProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build()), null); } @@ -508,7 +500,7 @@ public class DefaultDeviceStateService implements DeviceStateService { .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(stateData.getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - queueProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index adff35fa51..d6e04d6dd3 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -22,7 +22,6 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; -import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; @@ -34,21 +33,22 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; @@ -86,16 +86,16 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer private TbServiceInfoProvider serviceInfoProvider; @Autowired - private TbCoreQueueProvider coreQueueProvider; + private TbQueueProducerProvider producerProvider; @Autowired - private LocalSubscriptionService localSubscriptionService; + private TbLocalSubscriptionService localSubscriptionService; @Autowired private DeviceStateService deviceStateService; @Autowired - private ActorService actorService; + private TbClusterService clusterService; private final Map> subscriptionsByEntityId = new ConcurrentHashMap<>(); private final Map> subscriptionsByWsSessionId = new ConcurrentHashMap<>(); @@ -104,13 +104,13 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer private ExecutorService tsCallBackExecutor; private String serviceId; - private TbQueueProducer> toCoreProducer; + private TbQueueProducer> toCoreNotificationsProducer; @PostConstruct public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-sub-callback")); serviceId = serviceInfoProvider.getServiceId(); - toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + toCoreNotificationsProducer = producerProvider.getTbCoreNotificationsMsgProducer(); } @PreDestroy @@ -241,9 +241,8 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } } } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { - DeviceAttributesEventNotificationMsg notificationMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)); - actorService.onMsg(notificationMsg); + clusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))); } } callback.onSuccess(); @@ -263,7 +262,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbMsgCallback.EMPTY); } else { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); - toCoreProducer.send(tpi, toProto(s, subscriptionUpdate), null); + toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); } } }); @@ -309,7 +308,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer }); if (!missedUpdates.isEmpty()) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); } }, e -> log.error("Failed to fetch missed updates.", e), tsCallBackExecutor); @@ -333,7 +332,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer missedUpdates -> { if (missedUpdates != null && !missedUpdates.isEmpty()) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId()); - toCoreProducer.send(tpi, toProto(subscription, missedUpdates), null); + toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null); } }, e -> log.error("Failed to fetch missed updates.", e), @@ -341,7 +340,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } } - private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { + private TbProtoQueueMsg toProto(TbSubscription subscription, List updates) { TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder(); builder.setSessionId(subscription.getSessionId()); @@ -367,7 +366,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer builder.addData(dataBuilder.build()); }); - ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToLocalSubscriptionServiceMsg( + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg( LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()) .build(); return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java rename to application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 89f745260a..1816410ff2 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -33,10 +33,10 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; @@ -53,7 +53,7 @@ import java.util.stream.Collectors; @Slf4j @Service -public class DefaultLocalSubscriptionService implements LocalSubscriptionService { +public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionService { private final Set currentPartitions = ConcurrentHashMap.newKeySet(); private final Map> subscriptionsBySessionId = new ConcurrentHashMap<>(); @@ -68,7 +68,7 @@ public class DefaultLocalSubscriptionService implements LocalSubscriptionService private PartitionService partitionService; @Autowired - private TbCoreQueueProvider coreQueueProvider; + private TbQueueProducerProvider producerProvider; @Autowired @Lazy @@ -80,7 +80,7 @@ public class DefaultLocalSubscriptionService implements LocalSubscriptionService @PostConstruct public void initExecutor() { wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); - toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java rename to application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java index a71155f0ce..b6bee14a32 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/LocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -20,7 +20,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; -public interface LocalSubscriptionService { +public interface TbLocalSubscriptionService { void addSubscription(TbSubscription subscription); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ab2803345a..041be297ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -39,10 +39,10 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -73,7 +73,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private TimeseriesService tsService; @Autowired - private TbCoreQueueProvider coreQueueProvider; + private TbQueueProducerProvider producerProvider; @Autowired private PartitionService partitionService; @@ -90,7 +90,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); - toCoreProducer = coreQueueProvider.getTbCoreMsgProducer(); + toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 51556602f1..876ea4f2eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -49,7 +49,7 @@ import org.thingsboard.server.service.security.ValidationResult; import org.thingsboard.server.service.security.ValidationResultCode; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.subscription.LocalSubscriptionService; +import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; import org.thingsboard.server.service.subscription.TbAttributeSubscription; import org.thingsboard.server.service.subscription.TbTimeseriesSubscription; @@ -98,7 +98,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private final ConcurrentMap wsSessionsMap = new ConcurrentHashMap<>(); @Autowired - private LocalSubscriptionService subService; + private TbLocalSubscriptionService subService; @Autowired private TelemetryWebSocketMsgEndpoint msgEndpoint; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index fb369f4564..52d748b96b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -23,10 +23,10 @@ import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import java.util.UUID; import java.util.function.Consumer; @@ -43,8 +43,8 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService @Value("${queue.transport.notifications_topic}") private String notificationsTopic; - public DefaultTbCoreToTransportService(TbCoreQueueProvider tbCoreQueueProvider) { - this.tbTransportProducer = tbCoreQueueProvider.getTransportNotificationsMsgProducer(); + public DefaultTbCoreToTransportService(TbQueueProducerProvider tbQueueProducerProvider) { + this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); } @Override diff --git a/application/src/main/proto/cluster.proto b/application/src/main/proto/cluster.proto deleted file mode 100644 index f0ca4d6d7c..0000000000 --- a/application/src/main/proto/cluster.proto +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; -package cluster; - -option java_package = "org.thingsboard.server.gen.cluster"; -option java_outer_classname = "ClusterAPIProtos"; - -service ClusterRpcService { - rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {} -} - -message ClusterMessage { - MessageType messageType = 1; - MessageMataInfo messageMetaInfo = 2; - ServerAddress serverAddress = 3; - bytes payload = 4; -} - -message ServerAddress { - string host = 1; - int32 port = 2; -} - -message MessageMataInfo { - string payloadMetaInfo = 1; - repeated string tags = 2; -} - -enum MessageType { - - //Cluster control messages - RPC_SESSION_CREATE_REQUEST_MSG = 0; - TO_ALL_NODES_MSG = 1; - RPC_SESSION_TELL_MSG = 2; - RPC_BROADCAST_MSG = 3; - CONNECT_RPC_MESSAGE =4; - - CLUSTER_ACTOR_MESSAGE = 5; - // Messages related to TelemetrySubscriptionService - CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE = 6; - CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE = 7; - CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE = 8; - CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE = 9; - CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE = 10; - CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE = 11; - CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE = 12; - - CLUSTER_DEVICE_STATE_SERVICE_MESSAGE = 13; - CLUSTER_TRANSACTION_SERVICE_MESSAGE = 14; -} - -// Messages related to CLUSTER_TELEMETRY_MESSAGE - - -message FromDeviceRPCResponseProto { - int64 requestIdMSB = 1; - int64 requestIdLSB = 2; - string response = 3; - int32 error = 4; -} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c73e623d56..7a80a26713 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -540,6 +540,8 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -548,6 +550,8 @@ queue: print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -561,6 +565,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" transport: + # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 265a552efb..215bd85930 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -150,7 +150,8 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule device.getId(), new TbMsgMetaData(), TbMsgDataType.JSON, "{}", null, null, null); - actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); + //TODO 2.5 +// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); @@ -266,7 +267,8 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule new TbMsgMetaData(), TbMsgDataType.JSON, "{}", null, null, null); - actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); + //TODO 2.5 +// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index a4360d51e9..43cdbe6ada 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -143,7 +143,8 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac TbMsgDataType.JSON, "{}", null, null, null); - actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); + //TODO 2.5 +// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); Thread.sleep(3000); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 445465f5df..2634d5af99 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -26,9 +26,9 @@ import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.discovery.ConsistentHashPartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.ArrayList; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 9153e0fc96..dec6f6b50b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; /** @@ -26,9 +27,9 @@ public enum MsgType { /** * ADDED/UPDATED/DELETED events for server nodes. * - * See {@link org.thingsboard.server.common.msg.cluster.ClusterEventMsg} + * See {@link PartitionChangeMsg} */ - CLUSTER_EVENT_MSG, + PARTITION_CHANGE_MSG, APP_INIT_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 062eb02749..7c7816900b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index a52c237f87..d05948206c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -33,7 +33,7 @@ import java.util.Optional; * @author Andrew Shvayka */ @ToString -public class ComponentLifecycleMsg implements TbActorMsg, TenantAwareMsg, ToAllNodesMsg { +public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { @Getter private final TenantId tenantId; @Getter diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java similarity index 72% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java index 56f928a1c8..e9b1a890c9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ClusterEventMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java @@ -13,23 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.common.msg.queue; import lombok.Data; +import lombok.Getter; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import java.util.Set; + /** * @author Andrew Shvayka */ @Data -public final class ClusterEventMsg implements TbActorMsg { +public final class PartitionChangeMsg implements TbActorMsg { - private final ServerAddress serverAddress; - private final boolean added; + @Getter + private final ServiceKey serviceKey; + @Getter + private final Set partitions; @Override public MsgType getMsgType() { - return MsgType.CLUSTER_EVENT_MSG; + return MsgType.PARTITION_CHANGE_MSG; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java index 3396078124..b68717730b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceKey.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.common.msg.queue; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index a5de33c436..a24a8c4b7c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.common.msg.queue; public enum ServiceType { TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 1164465ed7..02ad1d0528 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 1a5c98ec80..1b2f984093 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index e7ea35b836..fa9d91d149 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; public interface TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index a2e92e112b..34ad3016b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -28,7 +28,7 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index 3e70040e2a..4fdd3bdf3c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -23,7 +23,7 @@ import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.UUID; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index fb379c213a..e747b2f99d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceKey; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 92b5723b1e..fdf79d4e1b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -24,6 +24,9 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; @@ -156,8 +159,6 @@ public class ConsistentHashPartitionService implements PartitionService { Set tpiList = partitions.stream() .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) .collect(Collectors.toSet()); - // Adding notifications topic for every @TopicPartitionInfo list - tpiList.add(getNotificationsTopic(serviceKey.getServiceType(), serviceInfoProvider.getServiceId())); applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); } }); @@ -184,16 +185,19 @@ public class ConsistentHashPartitionService implements PartitionService { } } - private Map> getServiceKeyListMap(List services) { - final Map> currentMap = new HashMap<>(); - services.forEach(serviceInfo -> { - for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { - ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + @Override + public Set getAllServiceIds(ServiceType serviceType) { + Set result = new HashSet<>(); + ServiceInfo current = serviceInfoProvider.getServiceInfo(); + if (current.getServiceTypesList().contains(serviceType.name())) { + result.add(current.getServiceId()); + } + for (ServiceInfo serviceInfo : currentOtherServices) { + if (serviceInfo.getServiceTypesList().contains(serviceType.name())) { + result.add(serviceInfo.getServiceId()); } - }); - return currentMap; + } + return result; } @Override @@ -201,17 +205,29 @@ public class ConsistentHashPartitionService implements PartitionService { switch (serviceType) { case TB_CORE: return tbCoreNotificationTopics.computeIfAbsent(serviceId, - id -> buildTopicPartitionInfo(serviceType, serviceId)); + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); case TB_RULE_ENGINE: return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, - id -> buildTopicPartitionInfo(serviceType, serviceId)); + id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); default: - return buildTopicPartitionInfo(serviceType, serviceId); + return buildNotificationsTopicPartitionInfo(serviceType, serviceId); } } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return new TopicPartitionInfo(serviceType.name().toLowerCase() + "." + serviceId, null, null, false); + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); + services.forEach(serviceInfo -> { + for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { + ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); + ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + } + }); + return currentMap; + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { + return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); } private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index cb7b04df44..2ced989525 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import javax.annotation.PostConstruct; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index 13976232e5..05218c68d4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; +import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 4eb4ca9a69..b22b53f93e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.List; @@ -38,12 +40,19 @@ public interface PartitionService { */ void recalculatePartitions(TransportProtos.ServiceInfo currentService, List otherServices); + /** + * Get all active service ids by service type + * @param serviceType to filter the list of services + * @return list of all active services + */ + Set getAllServiceIds(ServiceType serviceType); + /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. * This topic is likely to have single partition, and is always assigned to the service. - * @param tbCore + * @param serviceType * @param serviceId * @return */ - TopicPartitionInfo getNotificationsTopic(ServiceType tbCore, String serviceId); + TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 764f646a8a..8e20b35bde 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.discovery; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import java.util.List; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java index 4d02647fd0..2d51bc581a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.AllArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import java.util.Objects; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index ec08d69be7..d43d2a8860 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -24,7 +24,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.io.IOException; import java.time.Duration; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 8f177ecb3f..b749625909 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -27,7 +27,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Properties; import java.util.stream.Collectors; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index c0849d083e..8f226ff2e9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.memory; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index c235f876ce..a2fd7450ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -19,7 +19,7 @@ import lombok.Data; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @Data public class InMemoryTbQueueProducer implements TbQueueProducer { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java index b734562a61..5e1bab5e69 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java @@ -18,6 +18,13 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; @@ -25,11 +32,6 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -87,4 +89,24 @@ public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRul public TbQueueProducer> getTransportApiResponseProducer() { return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic() + ".notifications"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic() + ".notifications"); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic() + ".notifications"); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java index 3e33e4daa6..17e38b244e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java @@ -39,17 +39,17 @@ import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class InMemoryTransportQueueProvider implements TransportQueueProvider { +public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvider { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryTransportQueueProvider(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + public InMemoryTbTransportQueueProvider(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index 990baca5a5..56cf4fb55f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -17,8 +17,11 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; @@ -29,6 +32,7 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; @@ -38,6 +42,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; @@ -45,14 +50,13 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private TbQueueProducer> tbCoreProducer; - - public KafkaMonolithQueueProvider(TbKafkaSettings kafkaSettings, + public KafkaMonolithQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -79,21 +83,31 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return requestBuilder.build(); } - //TODO 2.5 Singleton + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + return requestBuilder.build(); + } + @Override public TbQueueProducer> getTbCoreMsgProducer() { - if (tbCoreProducer == null) { - synchronized (this) { - if (tbCoreProducer == null) { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - tbCoreProducer = requestBuilder.build(); - } - } - } - return tbCoreProducer; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override @@ -107,6 +121,17 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return consumerBuilder.build(); } + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> getToCoreMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); @@ -118,6 +143,17 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn return consumerBuilder.build(); } + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> getTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 0d6d62c3cb..177f5f3c3d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -17,6 +17,14 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; @@ -24,11 +32,7 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; @@ -38,6 +42,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { + private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; @@ -45,14 +50,13 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private TbQueueProducer> tbCoreProducer; - - public KafkaTbCoreQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbCoreQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -79,20 +83,31 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { return requestBuilder.build(); } + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + return requestBuilder.build(); + } + @Override public TbQueueProducer> getTbCoreMsgProducer() { - if (tbCoreProducer == null) { - synchronized (this) { - if (tbCoreProducer == null) { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(coreSettings.getTopic()); - tbCoreProducer = requestBuilder.build(); - } - } - } - return tbCoreProducer; + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); } @Override @@ -106,6 +121,17 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { return consumerBuilder.build(); } + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> getTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); @@ -125,4 +151,5 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index c75437f38b..36ffd05b48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -17,6 +17,12 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; @@ -24,9 +30,7 @@ import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; @@ -36,6 +40,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; @@ -43,12 +48,13 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbRuleEngineQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbRuleEngineQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -56,6 +62,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; } + @Override public TbQueueProducer> getTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); @@ -73,6 +80,17 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider requestBuilder.defaultTopic(coreSettings.getTopic()); return requestBuilder.build(); } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + return requestBuilder.build(); + } + + @Override public TbQueueProducer> getTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); @@ -82,14 +100,34 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider return requestBuilder.build(); } + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + return requestBuilder.build(); + } + @Override public TbQueueConsumer> getToRuleEngineMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-rule-engine-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("tb-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + return consumerBuilder.build(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java similarity index 91% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java index 10cc9c1871..2213889964 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java @@ -41,7 +41,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class KafkaTransportQueueProvider implements TransportQueueProvider { +public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; @@ -50,12 +50,12 @@ public class KafkaTransportQueueProvider implements TransportQueueProvider { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTransportQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbTransportQueueProvider(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java new file mode 100644 index 0000000000..c83159242d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { + + private final TbCoreQueueProvider tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + public TbCoreQueueProducerProvider(TbCoreQueueProvider tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java index 9fdec01830..bbbef16f65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -44,6 +45,13 @@ public interface TbCoreQueueProvider { */ TbQueueProducer> getRuleEngineMsgProducer(); + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + /** * Used to push messages to other instances of TB Core Service * @@ -51,6 +59,13 @@ public interface TbCoreQueueProvider { */ TbQueueProducer> getTbCoreMsgProducer(); + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** * Used to consume messages by TB Core Service * @@ -58,6 +73,13 @@ public interface TbCoreQueueProvider { */ TbQueueConsumer> getToCoreMsgConsumer(); + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToCoreNotificationsMsgConsumer(); + /** * Used to consume Transport API Calls * diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java new file mode 100644 index 0000000000..34c6ee577f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for providing various Producers to other services. + */ +public interface TbQueueProducerProvider { + + /** + * Used to push messages to instances of TB Transport Service + * + * @return + */ + TbQueueProducer> getTransportNotificationsMsgProducer(); + + /** + * Used to push messages to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineMsgProducer(); + + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java new file mode 100644 index 0000000000..ed7dd43f2a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { + + private final TbRuleEngineQueueProvider tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + private TbQueueProducer> toRuleEngineNotifications; + private TbQueueProducer> toTbCoreNotifications; + + + public TbRuleEngineProducerProvider(TbRuleEngineQueueProvider tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return toTransport; + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return toRuleEngineNotifications; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java index c4d610cab1..fafa7da9e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. @@ -42,6 +44,13 @@ public interface TbRuleEngineQueueProvider { */ TbQueueProducer> getRuleEngineMsgProducer(); + /** + * Used to push notifications to instances of TB RuleEngine Service + * + * @return + */ + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + /** * Used to push messages to other instances of TB Core Service * @@ -49,6 +58,13 @@ public interface TbRuleEngineQueueProvider { */ TbQueueProducer> getTbCoreMsgProducer(); + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbCoreNotificationsMsgProducer(); + /** * Used to consume messages by TB Core Service * @@ -56,4 +72,11 @@ public interface TbRuleEngineQueueProvider { */ TbQueueConsumer> getToRuleEngineMsgConsumer(); + /** + * Used to consume high priority messages by TB Core Service + * + * @return + */ + TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java new file mode 100644 index 0000000000..d78f10ef74 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +//TODO 2.5 Maybe remove this service if it is not used. +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { + + private final TbTransportQueueProvider tbQueueProvider; + private TbQueueProducer> toTransport; + private TbQueueProducer> toRuleEngine; + private TbQueueProducer> toTbCore; + + public TbTransportQueueProducerProvider(TbTransportQueueProvider tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); + this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return toRuleEngine; + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return toTbCore; + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java index 2aa18a62ca..09521692f6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java @@ -25,7 +25,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -public interface TransportQueueProvider { +public interface TbTransportQueueProvider { TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate(); diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 18c47e542b..87c256174c 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -337,6 +337,12 @@ message LocalSubscriptionServiceMsgProto { TbSubscriptionUpdateProto subUpdate = 1; } +message FromDeviceRPCResponseProto { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string response = 3; + int32 error = 4; +} /** * Main messages; */ @@ -359,7 +365,14 @@ message ToCoreMsg { TransportToDeviceActorMsg toDeviceActorMsg = 1; DeviceStateServiceMsgProto deviceStateServiceMsg = 2; SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; - LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 4; + bytes toDeviceActorNotificationMsg = 4; +} + +/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ +message ToCoreNotificationMsg { + LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; + bytes componentLifecycleMsg = 3; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -369,6 +382,11 @@ message ToRuleEngineMsg { bytes tbMsg = 3; } +message ToRuleEngineNotificationMsg { + bytes componentLifecycleMsg = 1; + FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; +} + /* Messages that are handled by ThingsBoard Transport Service */ message ToTransportMsg { DeviceActorToTransportMsg toDeviceSessionMsg = 1; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index 9323cfa7a0..fd4c8e2d95 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -20,14 +20,8 @@ import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.transport.SessionMsgProcessor; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; -import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 42d3a534a9..9b8c23bf8b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,7 +17,6 @@ package org.thingsboard.server.common.transport.service; import com.google.gson.Gson; import com.google.gson.JsonObject; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -44,9 +43,9 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.ServiceType; -import org.thingsboard.server.queue.discovery.TopicPartitionInfo; -import org.thingsboard.server.queue.provider.TransportQueueProvider; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbTransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -54,18 +53,15 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportToRuleEngineMsg; import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -95,7 +91,7 @@ public class DefaultTransportService implements TransportService { private int notificationsPollDuration; private final Gson gson = new Gson(); - private final TransportQueueProvider queueProvider; + private final TbTransportQueueProvider queueProvider; private final PartitionService partitionService; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; @@ -114,7 +110,7 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TransportQueueProvider queueProvider, PartitionService partitionService) { + public DefaultTransportService(TbTransportQueueProvider queueProvider, PartitionService partitionService) { this.queueProvider = queueProvider; this.partitionService = partitionService; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java index 34bfe4d3f4..7cab689d13 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceRpcRequest.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.api; import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -28,11 +29,11 @@ import java.util.UUID; @Builder public final class RuleEngineDeviceRpcRequest { + private final TenantId tenantId; private final DeviceId deviceId; private final int requestId; private final UUID requestUUID; - private final String originHost; - private final int originPort; + private final String originServiceId; private final boolean oneway; private final String method; private final String body; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index 69ce133c5d..5b76ecf624 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -23,8 +23,8 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReply(DeviceId deviceId, int requestId, String body); + void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body); - void sendRpcRequest(RuleEngineDeviceRpcRequest request, Consumer consumer); + void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java index 26f7d14f8b..85fc75139b 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.concurrent.ExecutionException; @@ -31,6 +31,6 @@ public interface TbNode { void destroy(); - default void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) {} + default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {} } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index cd566f8dd2..93114c28e0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -74,7 +74,7 @@ public class TbMsgGeneratorNode implements TbNode { } @Override - public void onClusterEventMsg(TbContext ctx, ClusterEventMsg msg) { + public void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) { updateGeneratorState(ctx); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 9444b52210..0c5c7d188e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -58,7 +58,7 @@ public class TbSendRPCReplyNode implements TbNode { } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReply(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.getRpcService().sendRpcReplyToDevice(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index 9aa94905e6..e4eeef2e5f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -16,8 +16,6 @@ package org.thingsboard.rule.engine.rpc; import com.datastax.driver.core.utils.UUIDs; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -38,7 +36,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.io.IOException; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -86,10 +83,8 @@ public class TbSendRPCRequestNode implements TbNode { tmp = msg.getMetaData().getValue("requestUUID"); UUID requestUUID = !StringUtils.isEmpty(tmp) ? UUID.fromString(tmp) : UUIDs.timeBased(); - tmp = msg.getMetaData().getValue("originHost"); - String originHost = !StringUtils.isEmpty(tmp) ? tmp : null; - tmp = msg.getMetaData().getValue("originPort"); - int originPort = !StringUtils.isEmpty(tmp) ? Integer.parseInt(tmp) : 0; + tmp = msg.getMetaData().getValue("originServiceId"); + String originServiceId = !StringUtils.isEmpty(tmp) ? tmp : null; tmp = msg.getMetaData().getValue("expirationTime"); long expirationTime = !StringUtils.isEmpty(tmp) ? Long.parseLong(tmp) : (System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(config.getTimeoutInSeconds())); @@ -106,16 +101,16 @@ public class TbSendRPCRequestNode implements TbNode { .oneway(oneway) .method(json.get("method").getAsString()) .body(params) + .tenantId(ctx.getTenantId()) .deviceId(new DeviceId(msg.getOriginator().getId())) .requestId(requestId) .requestUUID(requestUUID) - .originHost(originHost) - .originPort(originPort) + .originServiceId(originServiceId) .expirationTime(expirationTime) .restApiCall(restApiCall) .build(); - ctx.getRpcService().sendRpcRequest(request, ruleEngineDeviceRpcResponse -> { + ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); ctx.tellNext(next, TbRelationTypes.SUCCESS); From 5cf35dae945d69ba2e7ee9b28686389134369018 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Thu, 26 Mar 2020 14:47:24 +0200 Subject: [PATCH 027/196] Add delete translate and change setting name (#2546) --- ui/src/app/components/gateway/gateway-form.directive.js | 2 +- ui/src/app/locale/locale.constant-en_US.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/src/app/components/gateway/gateway-form.directive.js b/ui/src/app/components/gateway/gateway-form.directive.js index 1e2e02ce24..5efa2ee121 100644 --- a/ui/src/app/components/gateway/gateway-form.directive.js +++ b/ui/src/app/components/gateway/gateway-form.directive.js @@ -119,7 +119,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, vm.ctx.widgetTitle = widgetTitle; archiveFileName = vm.settings.archiveFileName && vm.settings.archiveFileName.length ? vm.settings.archiveFileName : 'gatewayConfiguration'; - gatewayNameExists = utils.customTranslation(vm.settings.deviceNameExist, vm.settings.deviceNameExist) || $translate.instant('gateway.gateway-exists'); + gatewayNameExists = utils.customTranslation(vm.settings.gatewayNameExists, vm.settings.gatewayNameExists) || $translate.instant('gateway.gateway-exists'); successfulSaved = utils.customTranslation(vm.settings.successfulSave, vm.settings.successfulSave) || $translate.instant('gateway.gateway-saved'); } diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index f0dc673f84..a0750164ca 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -50,7 +50,8 @@ "export": "Export", "share-via": "Share via {{provider}}", "continue": "Continue", - "discard-changes": "Discard Changes" + "discard-changes": "Discard Changes", + "download": "Download" }, "aggregation": { "aggregation": "Aggregation", From dfd5ab394264357782d3295872a181a409af96d6 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 26 Mar 2020 21:51:01 +0900 Subject: [PATCH 028/196] tools/src/main/shell/*.sh: Expand tabs for consistency (#2548) --- tools/src/main/shell/client.keygen.sh | 2 +- tools/src/main/shell/server.keygen.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/src/main/shell/client.keygen.sh b/tools/src/main/shell/client.keygen.sh index ed35dcaa7e..d4e750dc87 100755 --- a/tools/src/main/shell/client.keygen.sh +++ b/tools/src/main/shell/client.keygen.sh @@ -20,7 +20,7 @@ usage() { echo "and imports server public key to client keystore" echo "usage: ./client.keygen.sh [-p file]" echo " -p | --props | --properties file Properties file. default value is ./keygen.properties" - echo " -h | --help | ? Show this message" + echo " -h | --help | ? Show this message" } PROPERTIES_FILE=keygen.properties diff --git a/tools/src/main/shell/server.keygen.sh b/tools/src/main/shell/server.keygen.sh index 2eae1cc31e..c3a7dbbca3 100755 --- a/tools/src/main/shell/server.keygen.sh +++ b/tools/src/main/shell/server.keygen.sh @@ -23,7 +23,7 @@ usage() { echo " -d | --dir directory Server keystore directory, where the generated keystore file will be copied. If specified, overrides the value from the properties file" echo " Default value is SERVER_KEYSTORE_DIR property from properties file" echo " -p | --props | --properties file Properties file. default value is ./keygen.properties" - echo " -h | --help | ? Show this message" + echo " -h | --help | ? Show this message" } COPY=true; @@ -155,4 +155,4 @@ if [[ $COPY = true ]]; then fi fi fi -echo "Done." \ No newline at end of file +echo "Done." From 6fe25a2ee29ee68e78999050b1ba3265a1466aa7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Sat, 28 Mar 2020 01:42:17 +0200 Subject: [PATCH 029/196] Refactoring to support TB-Rule-Engine service --- .../server/actors/ActorSystemContext.java | 26 +-- .../device/DeviceActorMessageProcessor.java | 23 +-- .../actors/ruleChain/RuleChainActor.java | 2 +- .../actors/ruleChain/RuleNodeActor.java | 2 + .../ThingsboardSecurityConfiguration.java | 1 + .../server/config/WebSocketConfiguration.java | 5 +- .../server/controller/AdminController.java | 14 +- .../server/controller/AlarmController.java | 2 + .../server/controller/AssetController.java | 2 + .../server/controller/AuditLogController.java | 2 + .../server/controller/AuthController.java | 2 + .../server/controller/BaseController.java | 3 + .../ComponentDescriptorController.java | 2 + .../server/controller/CustomerController.java | 2 + .../controller/DashboardController.java | 3 +- .../server/controller/DeviceController.java | 2 + .../controller/EntityRelationController.java | 4 +- .../controller/EntityViewController.java | 2 + .../server/controller/EventController.java | 2 + .../server/controller/RpcController.java | 3 +- .../controller/RuleChainController.java | 2 + .../controller/TelemetryController.java | 2 + .../server/controller/TenantController.java | 2 + .../server/controller/UserController.java | 2 + .../controller/WidgetTypeController.java | 2 + .../controller/WidgetsBundleController.java | 2 + .../controller/plugin/TbWebSocketHandler.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 155 ++++++------------ .../DefaultTbRuleEngineConsumerService.java | 154 +++++------------ .../processing/AbstractConsumerService.java | 150 +++++++++++++++++ .../rpc/DefaultTbCoreDeviceRpcService.java | 5 +- .../rpc/DefaultTbRuleEngineRpcService.java | 7 +- .../auth/rest/RestAuthenticationProvider.java | 2 + ...RestAwareAuthenticationFailureHandler.java | 1 + ...RestAwareAuthenticationSuccessHandler.java | 1 + .../device/DefaultDeviceAuthService.java | 13 +- .../security/model/token/JwtTokenFactory.java | 1 + .../permission/AccessControlService.java | 2 - .../DefaultAccessControlService.java | 1 + .../system/DefaultSystemSecurityService.java | 1 + .../DefaultDeviceSessionCacheService.java | 2 + .../state/DefaultDeviceStateService.java | 18 +- .../service/state/DeviceStateService.java | 2 +- .../DefaultSubscriptionManagerService.java | 2 + .../DefaultTbLocalSubscriptionService.java | 2 + .../DefaultTelemetrySubscriptionService.java | 47 ++++-- .../DefaultTelemetryWebSocketService.java | 2 + .../BaseRuleChainTransactionService.java | 10 +- .../DefaultTbCoreToTransportService.java | 3 +- .../transport/DefaultTransportApiService.java | 2 + ...ce.java => TbCoreTransportApiService.java} | 9 +- .../service/update/DefaultUpdateService.java | 2 + .../src/main/resources/thingsboard.yml | 11 +- .../server/common/msg/queue/ServiceKey.java | 2 + .../common/msg/queue/TopicPartitionInfo.java | 2 + .../provider/KafkaMonolithQueueProvider.java | 6 +- .../provider/KafkaTbCoreQueueProvider.java | 6 +- .../KafkaTbRuleEngineQueueProvider.java | 6 +- .../queue/provider/TbCoreQueueProvider.java | 8 +- .../server/queue/util/TbCoreComponent.java | 15 +- .../server/queue/util/TbKafkaQueue.java | 19 +-- .../queue/util/TbMonolithComponent.java | 19 +-- .../queue/util/TbMonolithOrCoreComponent.java | 22 +++ .../util/TbMonolithOrRuleEngineComponent.java | 22 +++ .../queue/util/TbRuleEngineComponent.java | 22 +++ .../transport/mqtt/MqttTransportHandler.java | 16 +- .../service/DefaultTransportService.java | 9 +- .../session/DeviceAwareSessionContext.java | 8 +- 68 files changed, 526 insertions(+), 380 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java rename application/src/main/java/org/thingsboard/server/service/transport/{RemoteTransportApiService.java => TbCoreTransportApiService.java} (93%) rename application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java => common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java (60%) rename application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java => common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java (53%) rename application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java => common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java (53%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 03b65e7ccc..bd7c25ddc9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -70,7 +70,6 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; @@ -125,10 +124,6 @@ public class ActorSystemContext { @Getter private DataDecodingEncodingService encodingService; - @Autowired - @Getter - private DeviceAuthService deviceAuthService; - @Autowired @Getter private DeviceService deviceService; @@ -212,10 +207,6 @@ public class ActorSystemContext { @Getter private MailExecutorService mailExecutor; - @Autowired - @Getter - private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor; - @Autowired @Getter private DbCallbackExecutorService dbCallbackExecutor; @@ -232,37 +223,34 @@ public class ActorSystemContext { @Getter private MailService mailService; - @Autowired + //TODO: separate context for TbCore and TbRuleEngine + @Autowired(required = false) @Getter private DeviceStateService deviceStateService; - @Autowired + @Autowired(required = false) @Getter private DeviceSessionCacheService deviceSessionCacheService; - @Lazy - @Autowired + @Autowired(required = false) @Getter private TbCoreToTransportService tbCoreToTransportService; /** * The following Service will be null if we operate in tb-core mode */ - @Lazy - @Getter @Autowired(required = false) + @Getter private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; /** * The following Service will be null if we operate in tb-rule-engine mode */ - @Lazy - @Getter @Autowired(required = false) + @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; - @Lazy - @Autowired + @Autowired(required = false) @Getter private RuleChainTransactionService ruleChainTransactionService; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index aff6200199..307797f864 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -212,8 +212,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { - //TODO 2.5 - boolean reportDeviceActivity = true; TransportToDeviceActorMsg msg = wrapper.getMsg(); TbMsgCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { @@ -225,37 +223,18 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasSubscribeToRPC()) { processSubscriptionCommands(context, msg.getSessionInfo(), msg.getSubscribeToRPC()); } -// if (msg.hasPostAttributes()) { -// handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes()); -// reportDeviceActivity = true; -// } -// if (msg.hasPostTelemetry()) { -// handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry()); -// reportDeviceActivity = true; -// } if (msg.hasGetAttributes()) { handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes()); } if (msg.hasToDeviceRPCCallResponse()) { processRpcResponses(context, msg.getSessionInfo(), msg.getToDeviceRPCCallResponse()); } -// if (msg.hasToServerRPCCallRequest()) { -// handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest()); -// reportDeviceActivity = true; -// } if (msg.hasSubscriptionInfo()) { handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo()); } - if (reportDeviceActivity) { - reportLogicalDeviceActivity(); - } callback.onSuccess(); } - private void reportLogicalDeviceActivity() { - systemContext.getDeviceStateService().onDeviceActivity(deviceId); - } - private void reportSessionOpen() { systemContext.getDeviceStateService().onDeviceConnect(deviceId); } @@ -469,6 +448,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (sessions.size() == 1) { reportSessionOpen(); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, System.currentTimeMillis()); dumpSessions(); } else if (msg.getEvent() == SessionEvent.CLOSED) { log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId); @@ -496,6 +476,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (subscriptionInfo.getRpcSubscription()) { rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo()); } + systemContext.getDeviceStateService().onDeviceActivity(deviceId, subscriptionInfo.getLastActivityTime()); dumpSessions(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 1b9923139a..015e7a008b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -53,7 +53,7 @@ public class RuleChainActor extends ComponentActor implements TbCoreConsumerService { @Value("${queue.core.poll_interval}") private long pollDuration; @@ -72,56 +73,33 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { @Value("${queue.core.stats.enabled:false}") private boolean statsEnabled; - private final ActorSystemContext actorContext; private final DeviceStateService stateService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; - private final DataDecodingEncodingService encodingService; - private final TbQueueConsumer> mainConsumer; - private final TbQueueConsumer> nfConsumer; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - private volatile ExecutorService mainConsumerExecutor; - private volatile ExecutorService notificationsConsumerExecutor; private volatile boolean stopped = false; public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, TbCoreDeviceRpcService tbCoreDeviceRpcService) { - this.mainConsumer = tbCoreQueueProvider.getToCoreMsgConsumer(); - this.nfConsumer = tbCoreQueueProvider.getToCoreNotificationsMsgConsumer(); - this.actorContext = actorContext; + super(actorContext, encodingService, + tbCoreQueueProvider.getToCoreMsgConsumer(), tbCoreQueueProvider.getToCoreNotificationsMsgConsumer()); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; - this.encodingService = encodingService; this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; } @PostConstruct public void init() { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-consumer")); - this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-notifications-consumer")); + super.init("tb-core-consumer", "tb-core-notifications-consumer"); } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_CORE) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - log.info("Subscribing to notifications: {}", mainConsumer.getTopic()); - this.nfConsumer.subscribe(); - launchNotificationsConsumer(); - launchMainConsumer(); - } - - private void launchMainConsumer() { + protected void launchMainConsumer() { + log.info("Launching main consumer"); mainConsumerExecutor.execute(() -> { while (!stopped) { try { @@ -134,6 +112,7 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { + log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); @@ -177,57 +156,38 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { }); } - private void launchNotificationsConsumer() { - notificationsConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = nfConsumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - pendingMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); - try { - ToCoreNotificationMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); - } else if (toCoreMsg.hasFromDeviceRpcResponse()) { - log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); - forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); - } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } - } catch (Throwable e) { - log.warn("[{}] Failed to process notification: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); - } - nfConsumer.commit(); - } catch (Exception e) { - log.warn("Failed to obtain notifications from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new notifications", e2); - } - } + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_CORE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception { + ToCoreNotificationMsg toCoreMsg = msg.getValue(); + if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreMsg.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); + } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); } - log.info("Tb Core Notifications Consumer stopped."); - }); + callback.onSuccess(); + } } private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbMsgCallback callback) { @@ -245,23 +205,6 @@ public class DefaultTbCoreConsumerService implements TbCoreConsumerService { } } - @PreDestroy - public void destroy() { - stopped = true; - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - if (nfConsumer != null) { - nfConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - if (notificationsConsumerExecutor != null) { - notificationsConsumerExecutor.shutdownNow(); - } - } - private void forwardToLocalSubMgrService(LocalSubscriptionServiceMsgProto msg, TbMsgCallback callback) { if (msg.hasSubUpdate()) { localSubscriptionService.onSubscriptionUpdate(msg.getSubUpdate().getSessionId(), TbSubscriptionUtils.fromProto(msg.getSubUpdate()), callback); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index b9d2b98d04..835da15ab7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -19,49 +19,42 @@ import akka.actor.ActorRef; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.provider.TbRuleEngineQueueProvider; +import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +@TbMonolithOrRuleEngineComponent @Slf4j -public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerService { +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { @Value("${queue.rule_engine.poll_interval}") private long pollDuration; @@ -70,96 +63,24 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS @Value("${queue.rule_engine.stats.enabled:false}") private boolean statsEnabled; - private final ActorSystemContext actorContext; - private final TbQueueConsumer> mainConsumer; - private final TbQueueConsumer> nfConsumer; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private final TbRuleEngineProcessingStrategyFactory factory; - private final DataDecodingEncodingService encodingService; - private volatile ExecutorService mainConsumerExecutor; - private volatile ExecutorService notificationsConsumerExecutor; - private volatile boolean stopped = false; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, ActorSystemContext actorContext, - DataDecodingEncodingService encodingService) { + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, + ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { + super(actorContext, encodingService, + tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(), tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer()); this.factory = factory; - this.mainConsumer = tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(); - this.nfConsumer = tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer(); - this.actorContext = actorContext; - this.encodingService = encodingService; } @PostConstruct public void init() { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); - this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-notifications-consumer")); + super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); this.factory.newInstance(); } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceKey().getServiceType() == ServiceType.TB_RULE_ENGINE) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - - @EventListener(ApplicationReadyEvent.class) - public void onApplicationEvent(ApplicationReadyEvent event) { - this.nfConsumer.subscribe(); - launchNotificationsConsumer(); - launchMainConsumer(); - } - - private void launchNotificationsConsumer() { - notificationsConsumerExecutor.execute(() -> { - while (!stopped) { - try { - List> msgs = nfConsumer.poll(pollDuration); - if (msgs.isEmpty()) { - continue; - } - ConcurrentMap> pendingMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - pendingMap.forEach((id, msg) -> { - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); - try { - ToRuleEngineNotificationMsg toRuleEngineMsg = msg.getValue(); - if (toRuleEngineMsg.getComponentLifecycleMsg() != null && !toRuleEngineMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toRuleEngineMsg.getComponentLifecycleMsg().toByteArray()); - if (actorMsg.isPresent()) { - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); - } - callback.onSuccess(); - } else { - callback.onSuccess(); - } - } catch (Throwable e) { - log.warn("[{}] Failed to process message: {}", id, msg, e); - callback.onFailure(e); - } - }); - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { - pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); - } - nfConsumer.commit(); - } catch (Exception e) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - }); - } - - private void launchMainConsumer() { + protected void launchMainConsumer() { mainConsumerExecutor.execute(() -> { while (!stopped) { try { @@ -184,6 +105,7 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS CountDownLatch processingTimeoutLatch = new CountDownLatch(1); allMap.forEach((id, msg) -> { + log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); try { ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); @@ -217,6 +139,36 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS }); } + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollDuration; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception { + ToRuleEngineNotificationMsg nfMsg = msg.getValue(); + if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); + if (actorMsg.isPresent()) { + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); + actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + } + callback.onSuccess(); + } else { + callback.onSuccess(); + } + } + private void forwardToRuleEngineActor(TenantId tenantId, ByteString tbMsgData, TbMsgCallback callback) { TbMsg tbMsg = TbMsg.fromBytes(tbMsgData.toByteArray(), callback); actorContext.getAppActor().tell(new QueueToRuleEngineMsg(tenantId, tbMsg), ActorRef.noSender()); @@ -233,20 +185,4 @@ public class DefaultTbRuleEngineConsumerService implements TbRuleEngineConsumerS } } - @PreDestroy - public void destroy() { - stopped = true; - if (mainConsumer != null) { - mainConsumer.unsubscribe(); - } - if (nfConsumer != null) { - nfConsumer.unsubscribe(); - } - if (mainConsumerExecutor != null) { - mainConsumerExecutor.shutdownNow(); - } - if (notificationsConsumerExecutor != null) { - notificationsConsumerExecutor.shutdownNow(); - } - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java new file mode 100644 index 0000000000..e99770faf8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -0,0 +1,150 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.EventListener; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.service.encoding.DataDecodingEncodingService; +import org.thingsboard.server.service.queue.MsgPackCallback; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractConsumerService implements ApplicationListener { + + protected volatile ExecutorService mainConsumerExecutor; + private volatile ExecutorService notificationsConsumerExecutor; + protected volatile boolean stopped = false; + + protected final ActorSystemContext actorContext; + protected final DataDecodingEncodingService encodingService; + protected final TbQueueConsumer> mainConsumer; + protected final TbQueueConsumer> nfConsumer; + + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> mainConsumer, TbQueueConsumer> nfConsumer) { + this.actorContext = actorContext; + this.encodingService = encodingService; + this.mainConsumer = mainConsumer; + this.nfConsumer = nfConsumer; + } + + public void init(String mainConsumerThreadName, String nfConsumerThreadName) { + this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceKey().getServiceType() == getServiceType()) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationEvent(ApplicationReadyEvent event) { + log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); + this.nfConsumer.subscribe(); + launchNotificationsConsumer(); + launchMainConsumer(); + } + + protected abstract ServiceType getServiceType(); + + protected abstract void launchMainConsumer(); + + protected abstract long getNotificationPollDuration(); + + protected abstract long getNotificationPackProcessingTimeout(); + + protected void launchNotificationsConsumer() { + notificationsConsumerExecutor.execute(() -> { + while (!stopped) { + try { + List> msgs = nfConsumer.poll(getNotificationPollDuration()); + if (msgs.isEmpty()) { + continue; + } + ConcurrentMap> pendingMap = msgs.stream().collect( + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); + ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + pendingMap.forEach((id, msg) -> { + log.info("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + try { + handleNotification(id, msg, callback); + } catch (Throwable e) { + log.warn("[{}] Failed to process notification: {}", id, msg, e); + callback.onFailure(e); + } + }); + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { + pendingMap.forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + } + nfConsumer.commit(); + } catch (Exception e) { + log.warn("Failed to obtain notifications from queue.", e); + try { + Thread.sleep(getNotificationPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new notifications", e2); + } + } + } + log.info("Tb Core Notifications Consumer stopped."); + }); + } + + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception; + + @PreDestroy + public void destroy() { + stopped = true; + if (mainConsumer != null) { + mainConsumer.unsubscribe(); + } + if (nfConsumer != null) { + nfConsumer.unsubscribe(); + } + if (mainConsumerExecutor != null) { + mainConsumerExecutor.shutdownNow(); + } + if (notificationsConsumerExecutor != null) { + notificationsConsumerExecutor.shutdownNow(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index ed55ee11df..cc9cc4dd10 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.queue.TbClusterService; import javax.annotation.PostConstruct; @@ -54,7 +55,7 @@ import java.util.function.Consumer; */ @Service @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +@TbMonolithOrCoreComponent public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private static final ObjectMapper json = new ObjectMapper(); @@ -79,7 +80,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { this.actorContext = actorContext; } - @Autowired + @Autowired(required = false) public void setTbRuleEngineRpcService(Optional tbRuleEngineRpcService) { this.tbRuleEngineRpcService = tbRuleEngineRpcService; } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index 6481f1c2b6..f2390b9629 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.rpc; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; @@ -31,6 +29,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; import org.thingsboard.server.service.queue.TbClusterService; import javax.annotation.PostConstruct; @@ -45,7 +44,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +@TbMonolithOrRuleEngineComponent @Slf4j public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcService { @@ -67,7 +66,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi this.serviceInfoProvider = serviceInfoProvider; } - @Autowired + @Autowired(required = false) public void setTbCoreRpcService(Optional tbCoreRpcService) { this.tbCoreRpcService = tbCoreRpcService; } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index e359befb30..ac5b3625f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.system.SystemSecurityService; @@ -46,6 +47,7 @@ import ua_parser.Client; import java.util.UUID; + @Component @Slf4j public class RestAuthenticationProvider implements AuthenticationProvider { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java index 726ee76a2d..406b078e6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java @@ -20,6 +20,7 @@ import org.springframework.security.core.AuthenticationException; import org.springframework.security.web.authentication.AuthenticationFailureHandler; import org.springframework.stereotype.Component; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index aa55818084..80cc8ec6e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -23,6 +23,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.web.WebAttributes; import org.springframework.security.web.authentication.AuthenticationSuccessHandler; import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtToken; diff --git a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java index c58a664790..f686cc2037 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/device/DefaultDeviceAuthService.java @@ -24,16 +24,21 @@ import org.thingsboard.server.common.transport.auth.DeviceAuthResult; import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultDeviceAuthService implements DeviceAuthService { - @Autowired - DeviceService deviceService; + private final DeviceService deviceService; - @Autowired - DeviceCredentialsService deviceCredentialsService; + private final DeviceCredentialsService deviceCredentialsService; + + public DefaultDeviceAuthService(DeviceService deviceService, DeviceCredentialsService deviceCredentialsService) { + this.deviceService = deviceService; + this.deviceCredentialsService = deviceCredentialsService; + } @Override public DeviceAuthResult process(DeviceCredentialsFilter credentialsFilter) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index ff22c00154..f8907b3188 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java index 40d19da764..5412ed6def 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/AccessControlService.java @@ -15,11 +15,9 @@ */ package org.thingsboard.server.service.security.permission; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; public interface AccessControlService { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index 9777af7b8e..be99cc1aae 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.*; diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java index 11857f25b6..ca6d1f531d 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserServiceImpl; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index 952c63c9b9..35c940ef86 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -21,6 +21,7 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import java.util.Collections; import java.util.UUID; @@ -31,6 +32,7 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; * Created by ashvayka on 29.10.18. */ @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 91a49aae1e..986c2cfae7 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -60,6 +60,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -89,6 +90,7 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; * Created by ashvayka on 01.05.18. */ @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultDeviceStateService implements DeviceStateService { @@ -109,7 +111,8 @@ public class DefaultDeviceStateService implements DeviceStateService { private final TimeseriesService tsService; private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; - private final TelemetrySubscriptionService tsSubService; + + private TelemetrySubscriptionService tsSubService; @Value("${state.defaultInactivityTimeoutInSec}") @Getter @@ -137,13 +140,17 @@ public class DefaultDeviceStateService implements DeviceStateService { public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, - TbQueueProducerProvider producerProvider, PartitionService partitionService, TelemetrySubscriptionService tsSubService) { + TbQueueProducerProvider producerProvider, PartitionService partitionService) { this.tenantService = tenantService; this.deviceService = deviceService; this.attributesService = attributesService; this.tsService = tsService; this.producerProvider = producerProvider; this.partitionService = partitionService; + } + + @Autowired + public void setTsSubService(TelemetrySubscriptionService tsSubService) { this.tsSubService = tsSubService; } @@ -188,9 +195,8 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onDeviceActivity(DeviceId deviceId) { - deviceLastReportedActivity.put(deviceId, System.currentTimeMillis()); - long lastReportedActivity = deviceLastReportedActivity.getOrDefault(deviceId, 0L); + public void onDeviceActivity(DeviceId deviceId, long lastReportedActivity) { + deviceLastReportedActivity.put(deviceId, lastReportedActivity); long lastSavedActivity = deviceLastSavedActivity.getOrDefault(deviceId, 0L); if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); @@ -494,7 +500,7 @@ public class DefaultDeviceStateService implements DeviceStateService { try { TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state) - , null, null, null); + , null, null, TbMsgCallback.EMPTY); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, stateData.getTenantId(), stateData.getDeviceId()); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 430dd77a36..c7aadda963 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -35,7 +35,7 @@ public interface DeviceStateService extends ApplicationListener currentPartitions = ConcurrentHashMap.newKeySet(); - @Autowired - private AttributesService attrService; - - @Autowired - private TimeseriesService tsService; - - @Autowired - private TbQueueProducerProvider producerProvider; - - @Autowired - private PartitionService partitionService; - - @Autowired - private SubscriptionManagerService subscriptionManagerService; + private final AttributesService attrService; + private final TimeseriesService tsService; + private final TbQueueProducerProvider producerProvider; + private final PartitionService partitionService; + private Optional subscriptionManagerService; private TbQueueProducer> toCoreProducer; private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; + public DefaultTelemetrySubscriptionService(AttributesService attrService, + TimeseriesService tsService, + TbQueueProducerProvider producerProvider, + PartitionService partitionService) { + this.attrService = attrService; + this.tsService = tsService; + this.producerProvider = producerProvider; + this.partitionService = partitionService; + } + + @Autowired(required = false) + public void setSubscriptionManagerService(Optional subscriptionManagerService) { + this.subscriptionManagerService = subscriptionManagerService; + } + @PostConstruct public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); @@ -158,7 +165,11 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); + } else { + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); @@ -168,7 +179,11 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + if (subscriptionManagerService.isPresent()) { + subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + } else { + log.warn("Possible misconfiguration because subscriptionManagerService is null!"); + } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 876ea4f2eb..b4c15ee8cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -43,6 +43,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.TenantRateLimitException; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.ValidationCallback; import org.thingsboard.server.service.security.ValidationResult; @@ -83,6 +84,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 27.03.18. */ @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService { diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index b9424d49d9..b402f88896 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -24,6 +24,8 @@ import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.cluster.ServerAddress; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import javax.annotation.PostConstruct; @@ -44,11 +46,11 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; @Service +@TbMonolithOrRuleEngineComponent @Slf4j public class BaseRuleChainTransactionService implements RuleChainTransactionService { - @Autowired - private DbCallbackExecutorService callbackExecutor; + private final DbCallbackExecutorService callbackExecutor; @Value("${actors.rule.transaction.queue_size}") private int finalQueueSize; @@ -61,6 +63,10 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ private ExecutorService timeoutExecutor; + public BaseRuleChainTransactionService(DbCallbackExecutorService callbackExecutor) { + this.callbackExecutor = callbackExecutor; + } + @PostConstruct public void init() { timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index 52d748b96b..ab257f00f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import java.util.UUID; import java.util.function.Consumer; @@ -35,7 +36,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Slf4j @Service -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +@TbMonolithOrCoreComponent public class DefaultTbCoreToTransportService implements TbCoreToTransportService { private final TbQueueProducer> tbTransportProducer; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 42ee4c4619..dbff5bda86 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -52,6 +53,7 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service +@TbMonolithOrCoreComponent public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java rename to application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 4c9c801e83..446f7ba81a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -28,6 +28,8 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,9 +40,8 @@ import java.util.concurrent.*; */ @Slf4j @Service -//TODO 2.5: This Confitional annotation should be removed, and Service renamed to something meaningful -//@ConditionalOnProperty(prefix = "transport", value = "type", havingValue = "remote") -public class RemoteTransportApiService { +@TbMonolithOrCoreComponent +public class TbCoreTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; private final TransportApiService transportApiService; @@ -58,7 +59,7 @@ public class RemoteTransportApiService { private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; - public RemoteTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { + public TbCoreTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { this.tbCoreQueueProvider = tbCoreQueueProvider; this.transportApiService = transportApiService; } diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 1b8173c424..8844c1fe5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; +import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,6 +39,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service +@TbMonolithOrCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7a80a26713..dd9e6ad3ec 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,8 +205,6 @@ sql: # Actor system parameters actors: - cluster: - grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}" tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: @@ -370,7 +368,7 @@ spring: username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: - maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:50}" + maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:5}" # Audit log parameters audit-log: @@ -540,8 +538,6 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -550,8 +546,6 @@ queue: print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" rule_engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb.rule-engine.notifications}" poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -573,4 +567,5 @@ service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" - tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java index b68717730b..aa6eb27323 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java @@ -16,10 +16,12 @@ package org.thingsboard.server.common.msg.queue; import lombok.Getter; +import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; +@ToString public class ServiceKey { @Getter private final ServiceType serviceType; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 02ad1d0528..bf0fffe404 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -17,11 +17,13 @@ package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; +import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; import java.util.Optional; +@ToString public class TopicPartitionInfo { private final String topic; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index 56cf4fb55f..3566b29478 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -37,9 +36,12 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.util.TbKafkaQueue; +import org.thingsboard.server.queue.util.TbMonolithComponent; @Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") +@TbMonolithComponent +@TbKafkaQueue public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 177f5f3c3d..12aa66838e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -37,9 +36,12 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.queue.util.TbKafkaQueue; @Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") +@TbCoreComponent +@TbKafkaQueue public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index 36ffd05b48..f295f61d6a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -35,9 +34,12 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.util.TbKafkaQueue; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; @Component -@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") +@TbKafkaQueue +@TbRuleEngineComponent public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java index bbbef16f65..e74c07152b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -50,7 +50,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> getRuleEngineNotificationsMsgProducer(); /** * Used to push messages to other instances of TB Core Service @@ -64,7 +64,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); + TbQueueProducer> getTbCoreNotificationsMsgProducer(); /** * Used to consume messages by TB Core Service @@ -78,7 +78,7 @@ public interface TbCoreQueueProvider { * * @return */ - TbQueueConsumer> getToCoreNotificationsMsgConsumer(); + TbQueueConsumer> getToCoreNotificationsMsgConsumer(); /** * Used to consume Transport API Calls diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java index 50b940ae48..244aae5394 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToTransportMsgEncoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -13,17 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.queue.util; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.queue.kafka.TbKafkaEncoder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -/** - * Created by ashvayka on 05.10.18. - */ -public class ToTransportMsgEncoder implements TbKafkaEncoder { - @Override - public byte[] encode(ToTransportMsg value) { - return value.toByteArray(); - } +@ConditionalOnExpression("'${service.type:null}'=='tb-core'") +public @interface TbCoreComponent { } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java index 71739c0439..8d4af73492 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/ToRuleEngineMsgDecoder.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java @@ -13,21 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.transport; +package org.thingsboard.server.queue.util; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.kafka.TbKafkaDecoder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import java.io.IOException; - -/** - * Created by ashvayka on 05.10.18. - */ -public class ToRuleEngineMsgDecoder implements TbKafkaDecoder { - - @Override - public ToRuleEngineMsg decode(TbQueueMsg msg) throws IOException { - return ToRuleEngineMsg.parseFrom(msg.getData()); - } +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public @interface TbKafkaQueue { } diff --git a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java index f4b14144fc..c6dd9ebfff 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/ClusterRpcCallbackExecutorService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java @@ -13,21 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.executors; +package org.thingsboard.server.queue.util; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.AbstractListeningExecutor; - -@Component -public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor { - - @Value("${actors.cluster.grpc_callback_thread_pool_size}") - private int grpcCallbackExecutorThreadPoolSize; - - @Override - protected int getThreadPollSize() { - return grpcCallbackExecutorThreadPoolSize; - } +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +@ConditionalOnExpression("'${service.type:null}'=='monolith'") +public @interface TbMonolithComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java new file mode 100644 index 0000000000..a0165e3386 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") +public @interface TbMonolithOrCoreComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java new file mode 100644 index 0000000000..83e8a2d87a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") +public @interface TbMonolithOrRuleEngineComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java new file mode 100644 index 0000000000..9e79232906 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public @interface TbRuleEngineComponent { +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 0d96f94252..fb509e91af 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -410,13 +410,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDisconnect(ChannelHandlerContext ctx) { ctx.close(); log.info("[{}] Client disconnected!", sessionId); - if (deviceSessionCtx.isConnected()) { - transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); - transportService.deregisterSession(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); - } - } + doDisconnect(); } private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { @@ -485,9 +479,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void operationComplete(Future future) throws Exception { + doDisconnect(); + } + + private void doDisconnect() { if (deviceSessionCtx.isConnected()) { transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.CLOSED), null); transportService.deregisterSession(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDisconnect(); + } + deviceSessionCtx.setDisconnected(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 9b8c23bf8b..3fdf19c741 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueProvider; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -92,6 +93,7 @@ public class DefaultTransportService implements TransportService { private final Gson gson = new Gson(); private final TbTransportQueueProvider queueProvider; + private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; @@ -110,8 +112,9 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TbTransportQueueProvider queueProvider, PartitionService partitionService) { + public DefaultTransportService(TbTransportQueueProvider queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { this.queueProvider = queueProvider; + this.producerProvider = producerProvider; this.partitionService = partitionService; } @@ -126,8 +129,8 @@ public class DefaultTransportService implements TransportService { this.transportCallbackExecutor = Executors.newWorkStealingPool(20); this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); - ruleEngineMsgProducer = queueProvider.getRuleEngineMsgProducer(); - tbCoreMsgProducer = queueProvider.getTbCoreMsgProducer(); + ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); transportNotificationsConsumer.subscribe(); transportApiRequestTemplate.init(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java index 7c590ec925..c312b3a4d9 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/session/DeviceAwareSessionContext.java @@ -35,6 +35,7 @@ public abstract class DeviceAwareSessionContext implements SessionContext { private volatile DeviceId deviceId; @Getter private volatile DeviceInfoProto deviceInfo; + private volatile boolean connected; public DeviceId getDeviceId() { return deviceId; @@ -42,10 +43,15 @@ public abstract class DeviceAwareSessionContext implements SessionContext { public void setDeviceInfo(DeviceInfoProto deviceInfo) { this.deviceInfo = deviceInfo; + this.connected = true; this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB())); } public boolean isConnected() { - return deviceInfo != null; + return connected; + } + + public void setDisconnected() { + this.connected = false; } } From 903e459851fd98fd6ba592c279ceafb99712185e Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 26 Mar 2020 19:34:45 +0200 Subject: [PATCH 030/196] The map was not loaded with empty entity(not set longitude/latitude) --- ui/src/app/widget/lib/map-widget2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/app/widget/lib/map-widget2.js b/ui/src/app/widget/lib/map-widget2.js index 3d3fd4a38d..d1d6af490e 100644 --- a/ui/src/app/widget/lib/map-widget2.js +++ b/ui/src/app/widget/lib/map-widget2.js @@ -616,7 +616,7 @@ export default class TbMapWidgetV2 { locationChanged = true; } } - } else { + } else if (latData.length > 0 && lngData.length > 0) { // Create or update marker lat = latData[latData.length - 1][1]; lng = lngData[lngData.length - 1][1]; From c4269023dddd19c3b9c9f6002eabf51e350ec980 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Sat, 28 Mar 2020 02:30:37 +0200 Subject: [PATCH 031/196] Fixed Conditional annotations --- .../ThingsboardSecurityConfiguration.java | 1 - .../server/config/WebSocketConfiguration.java | 4 ++-- .../server/controller/AdminController.java | 4 ++-- .../server/controller/AlarmController.java | 4 ++-- .../server/controller/AssetController.java | 6 ++--- .../server/controller/AuditLogController.java | 4 ++-- .../server/controller/AuthController.java | 4 ++-- .../server/controller/BaseController.java | 5 ++--- .../ComponentDescriptorController.java | 4 ++-- .../server/controller/CustomerController.java | 4 ++-- .../controller/DashboardController.java | 4 ++-- .../server/controller/DeviceController.java | 4 ++-- .../controller/EntityRelationController.java | 4 ++-- .../controller/EntityViewController.java | 4 ++-- .../server/controller/EventController.java | 4 ++-- .../server/controller/RpcController.java | 4 ++-- .../controller/RuleChainController.java | 4 ++-- .../controller/TelemetryController.java | 4 ++-- .../server/controller/TenantController.java | 4 ++-- .../server/controller/UserController.java | 4 ++-- .../controller/WidgetTypeController.java | 4 ++-- .../controller/WidgetsBundleController.java | 5 ++--- .../controller/plugin/TbWebSocketHandler.java | 4 ++-- .../queue/DefaultTbCoreConsumerService.java | 21 +++++++++--------- .../DefaultTbRuleEngineConsumerService.java | 17 ++++++++------ .../processing/AbstractConsumerService.java | 20 +++++++---------- .../rpc/DefaultTbCoreDeviceRpcService.java | 6 ++--- .../rpc/DefaultTbRuleEngineRpcService.java | 4 ++-- .../auth/rest/RestAuthenticationProvider.java | 1 - ...RestAwareAuthenticationFailureHandler.java | 1 - ...RestAwareAuthenticationSuccessHandler.java | 1 - .../device/DefaultDeviceAuthService.java | 5 ++--- .../security/model/token/JwtTokenFactory.java | 1 - .../DefaultAccessControlService.java | 8 ------- .../system/DefaultSystemSecurityService.java | 1 - .../DefaultDeviceSessionCacheService.java | 5 ++--- .../state/DefaultDeviceStateService.java | 6 ++--- .../DefaultSubscriptionManagerService.java | 4 ++-- .../DefaultTbLocalSubscriptionService.java | 4 ++-- .../DefaultTelemetryWebSocketService.java | 4 ++-- .../BaseRuleChainTransactionService.java | 7 ++---- .../DefaultTbCoreToTransportService.java | 5 ++--- .../transport/DefaultTransportApiService.java | 4 ++-- .../transport/TbCoreTransportApiService.java | 3 +-- .../service/update/DefaultUpdateService.java | 4 ++-- application/src/main/resources/logback.xml | 1 + .../provider/KafkaMonolithQueueProvider.java | 6 ++--- .../provider/KafkaTbCoreQueueProvider.java | 6 ++--- .../KafkaTbRuleEngineQueueProvider.java | 6 ++--- .../server/queue/util/TbCoreComponent.java | 2 +- .../server/queue/util/TbKafkaQueue.java | 22 ------------------- .../queue/util/TbMonolithComponent.java | 22 ------------------- .../queue/util/TbMonolithOrCoreComponent.java | 22 ------------------- .../util/TbMonolithOrRuleEngineComponent.java | 22 ------------------- .../queue/util/TbRuleEngineComponent.java | 2 +- .../service/DefaultTransportService.java | 12 +++++----- 56 files changed, 113 insertions(+), 235 deletions(-) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 0921ba91e6..526f0b73c8 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -40,7 +40,6 @@ import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; import org.thingsboard.server.service.security.auth.jwt.JwtTokenAuthenticationProcessingFilter; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenAuthenticationProvider; diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 11cff0f2ac..e57695da69 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -32,13 +32,13 @@ import org.springframework.web.socket.server.support.HttpSessionHandshakeInterce import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.controller.plugin.TbWebSocketHandler; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; @Configuration -@TbMonolithOrCoreComponent +@TbCoreComponent @EnableWebSocket public class WebSocketConfiguration implements WebSocketConfigurer { diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 5603794d8b..8873d82366 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -30,14 +30,14 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.update.UpdateService; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api/admin") public class AdminController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 1412f29daf..c6f3fc3418 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -41,12 +41,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class AlarmController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 70164f8f20..03c5462446 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -32,17 +32,15 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -52,7 +50,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class AssetController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java index 314d2efebd..c99ba8a018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuditLogController.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.Arrays; import java.util.List; @@ -40,7 +40,7 @@ import java.util.UUID; import java.util.stream.Collectors; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class AuditLogController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 99b36296d4..6722254fd5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.common.data.security.model.SecuritySettings; @@ -57,7 +57,7 @@ import java.net.URI; import java.net.URISyntaxException; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") @Slf4j public class AuthController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index f9c3d53ef7..aead345a27 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -92,8 +92,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; -import org.thingsboard.server.queue.util.TbMonolithOrRuleEngineComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; @@ -114,7 +113,7 @@ import java.util.UUID; import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j -@TbMonolithOrCoreComponent +@TbCoreComponent public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; diff --git a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java index 5cc24c55b0..d75da8b34c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ComponentDescriptorController.java @@ -25,14 +25,14 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.HashSet; import java.util.List; import java.util.Set; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class ComponentDescriptorController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 6d2c5fedb4..18fcfc2ab5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -36,12 +36,12 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class CustomerController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index f0e2ffa176..68f18c9081 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -48,7 +48,7 @@ import java.util.HashSet; import java.util.Set; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class DashboardController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index b8a02b281a..2bfb1ed7e8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -51,7 +51,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -63,7 +63,7 @@ import java.util.List; import java.util.stream.Collectors; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class DeviceController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index e3ef8f4edb..484fa02066 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -33,14 +33,14 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class EntityRelationController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java index 1a48d4717b..ebe89bbd10 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityViewController.java @@ -47,7 +47,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -65,7 +65,7 @@ import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID; * Created by Victor Basanets on 8/28/2017. */ @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") @Slf4j public class EntityViewController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index 3943bd429f..500549c09a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -31,11 +31,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class EventController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index 3dc8db6c0d..731eeb4599 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -42,7 +42,7 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.rpc.RpcRequest; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.LocalRequestMetaData; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; @@ -60,7 +60,7 @@ import java.util.UUID; * Created by ashvayka on 22.03.18. */ @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping(TbUrlConstants.RPC_URL_PREFIX) @Slf4j public class RpcController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index f91d5540b6..364d348337 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -55,7 +55,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.script.JsInvokeService; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; @@ -69,7 +69,7 @@ import java.util.stream.Collectors; @Slf4j @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class RuleChainController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 31dadafd95..f97efffa92 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -70,7 +70,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -98,7 +98,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 22.03.18. */ @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping(TbUrlConstants.TELEMETRY_URL_PREFIX) @Slf4j public class TelemetryController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 11404cfccb..abd6bf6804 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -34,13 +34,13 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") @Slf4j public class TenantController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 881a76325a..a37d9c29b6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -56,7 +56,7 @@ import org.thingsboard.server.service.security.permission.Resource; import javax.servlet.http.HttpServletRequest; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class UserController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index ca29acdc1c..debe49b018 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -31,14 +31,14 @@ import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class WidgetTypeController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java index 1520683d34..3d5cd22400 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetsBundleController.java @@ -32,15 +32,14 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; @RestController -@TbMonolithOrCoreComponent +@TbCoreComponent @RequestMapping("/api") public class WidgetsBundleController extends BaseController { diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index b5d94399c1..f7d956e146 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.config.WebSocketConfiguration; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.telemetry.SessionEvent; @@ -53,7 +53,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; @Service -@TbMonolithOrCoreComponent +@TbCoreComponent @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index a9f0633eeb..a3613836f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; @@ -56,13 +56,12 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @Service -@TbMonolithOrCoreComponent +@TbCoreComponent @Slf4j public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { @@ -78,7 +77,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { while (!stopped) { try { @@ -144,15 +141,17 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { @@ -128,14 +128,17 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } mainConsumer.commit(); } catch (Exception e) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + if (!stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } + log.info("TB Rule Engine Consumer stopped."); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index e99770faf8..0a3ae8f251 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -117,15 +117,17 @@ public abstract class AbstractConsumerService> tbTransportProducer; diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index dbff5bda86..fdb8c86f70 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -41,7 +41,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -53,7 +53,7 @@ import java.util.concurrent.locks.ReentrantLock; */ @Slf4j @Service -@TbMonolithOrCoreComponent +@TbCoreComponent public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 446f7ba81a..7a5735b843 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.provider.TbCoreQueueProvider; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -40,7 +39,7 @@ import java.util.concurrent.*; */ @Slf4j @Service -@TbMonolithOrCoreComponent +@TbCoreComponent public class TbCoreTransportApiService { private final TbCoreQueueProvider tbCoreQueueProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index 8844c1fe5b..4b7f0f6366 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.UpdateMessage; -import org.thingsboard.server.queue.util.TbMonolithOrCoreComponent; +import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -39,7 +39,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Service -@TbMonolithOrCoreComponent +@TbCoreComponent @Slf4j public class DefaultUpdateService implements UpdateService { diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 3d47db432f..e5de845fbc 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,6 +25,7 @@ + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java index 3566b29478..56cf4fb55f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -36,12 +37,9 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.util.TbKafkaQueue; -import org.thingsboard.server.queue.util.TbMonolithComponent; @Component -@TbMonolithComponent -@TbKafkaQueue +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 12aa66838e..177f5f3c3d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -36,12 +37,9 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.queue.util.TbKafkaQueue; @Component -@TbCoreComponent -@TbKafkaQueue +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index f295f61d6a..36ffd05b48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -34,12 +35,9 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; -import org.thingsboard.server.queue.util.TbKafkaQueue; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; @Component -@TbKafkaQueue -@TbRuleEngineComponent +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { private final PartitionService partitionService; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java index 244aae5394..c42b9b9f7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -17,6 +17,6 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -@ConditionalOnExpression("'${service.type:null}'=='tb-core'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public @interface TbCoreComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java deleted file mode 100644 index 8d4af73492..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbKafkaQueue.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${queue.type:null}'=='kafka'") -public @interface TbKafkaQueue { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java deleted file mode 100644 index c6dd9ebfff..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithComponent.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${service.type:null}'=='monolith'") -public @interface TbMonolithComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java deleted file mode 100644 index a0165e3386..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrCoreComponent.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") -public @interface TbMonolithOrCoreComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java deleted file mode 100644 index 83e8a2d87a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbMonolithOrRuleEngineComponent.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.util; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; - -@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") -public @interface TbMonolithOrRuleEngineComponent { -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java index 9e79232906..855bebb19e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -17,6 +17,6 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") public @interface TbRuleEngineComponent { } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3fdf19c741..3891b197a6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -150,11 +150,13 @@ public class DefaultTransportService implements TransportService { }); transportNotificationsConsumer.commit(); } catch (Exception e) { - log.warn("Failed to obtain messages from queue.", e); - try { - Thread.sleep(notificationsPollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + if (!stopped) { + log.warn("Failed to obtain messages from queue.", e); + try { + Thread.sleep(notificationsPollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } From 2553cf6b6fcfb1ccfa8f8d0a5cfbfe1466990f8c Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 30 Mar 2020 19:13:34 +0300 Subject: [PATCH 032/196] created Aws Sqs Queue (#2534) * created Aws Sqs Queue * improvement AwsSqs providers * revert package-lock.json * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * aws improvements * aws improvements * aws improvements * added visibility timeout to aws queue --- .../server/controller/RpcController.java | 1 - .../processing/AbstractConsumerService.java | 9 + .../src/main/resources/thingsboard.yml | 8 +- common/queue/pom.xml | 5 + .../server/queue/TbQueueMsgDecoder.java | 24 ++ .../server/queue/TbQueueProducer.java | 1 + .../common/DefaultTbQueueRequestTemplate.java | 11 + .../DefaultTbQueueResponseTemplate.java | 12 +- .../queue/kafka/TBKafkaConsumerTemplate.java | 7 +- .../queue/kafka/TBKafkaProducerTemplate.java | 5 + .../queue/memory/InMemoryTbQueueProducer.java | 5 + .../provider/AwsSqsMonolithQueueProvider.java | 127 ++++++++++ .../provider/AwsSqsTbCoreQueueProvider.java | 117 +++++++++ .../AwsSqsTbRuleEngineQueueProvider.java | 97 +++++++ .../AwsSqsTransportQueueProvider.java | 93 +++++++ .../queue/sqs/AwsSqsTbQueueMsgMetadata.java | 28 ++ .../server/queue/sqs/TbAwsSqsAdmin.java | 65 +++++ .../queue/sqs/TbAwsSqsConsumerTemplate.java | 239 ++++++++++++++++++ .../server/queue/sqs/TbAwsSqsMsg.java | 38 +++ .../queue/sqs/TbAwsSqsProducerTemplate.java | 127 ++++++++++ .../server/queue/sqs/TbAwsSqsSettings.java | 44 ++++ .../service/DefaultTransportService.java | 7 + pom.xml | 6 + rule-engine/rule-engine-components/pom.xml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 6 +- 25 files changed, 1074 insertions(+), 10 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index 731eeb4599..b88c3f3695 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -88,7 +88,6 @@ public class RpcController extends BaseController { return handleDeviceRPCRequest(false, new DeviceId(UUID.fromString(deviceIdStr)), requestBody); } - private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 0a3ae8f251..781762a352 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -136,6 +136,15 @@ public abstract class AbstractConsumerServiceorg.apache.kafka kafka-clients + + com.amazonaws + aws-java-sdk-sqs + + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java new file mode 100644 index 0000000000..4cb38f6685 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueMsgDecoder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.thingsboard.server.queue.TbQueueMsg; + +public interface TbQueueMsgDecoder { + + T decode(TbQueueMsg msg) throws InvalidProtocolBufferException; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index fa9d91d149..ef308291a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -25,4 +25,5 @@ public interface TbQueueProducer { void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); + void stop(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 34ad3016b0..40ca2219dd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -92,6 +92,8 @@ public class DefaultTbQueueRequestTemplate responses = responseTemplate.poll(pollInterval); if (responses.size() > 0) { log.trace("Polling responses completed, consumer records count [{}]", responses.size()); + } else { + continue; } responses.forEach(response -> { log.trace("Received response to Kafka Template request: {}", response); @@ -145,6 +147,15 @@ public class DefaultTbQueueRequestTemplate requests = requestTemplate.poll(pollInterval); + if (requests.isEmpty()) { + continue; + } + requests.forEach(request -> { long currentTime = System.currentTimeMillis(); long requestTime = bytesToLong(request.getHeaders().get(REQUEST_TIME)); @@ -147,6 +151,12 @@ public class DefaultTbQueueResponseTemplate implements TbQueueCon } else { if (!subscribed) { List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(this::createTopicIfNotExists); + topicNames.forEach(admin::createTopicIfNotExists); consumer.subscribe(topicNames); subscribed = true; } @@ -130,7 +130,4 @@ public class TBKafkaConsumerTemplate implements TbQueueCon return decoder.decode(new KafkaTbQueueMsg(record)); } - private void createTopicIfNotExists(String topic) { - admin.createTopicIfNotExists(topic); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index b749625909..2fbfa7d3df 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -84,4 +84,9 @@ public class TBKafkaProducerTemplate implements TbQueuePro } }); } + + @Override + public void stop() { + + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index a2fd7450ee..cb1ff939d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -50,4 +50,9 @@ public class InMemoryTbQueueProducer implements TbQueuePro } } } + + @Override + public void stop() { + + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java new file mode 100644 index 0000000000..206a71d85e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java @@ -0,0 +1,127 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") +public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueAdmin admin; + + public AwsSqsMonolithQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + admin = new TbAwsSqsAdmin(sqsSettings); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java new file mode 100644 index 0000000000..052d3fc0e3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") +public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbAwsSqsSettings sqsSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + + private final TbQueueAdmin admin; + + public AwsSqsTbCoreQueueProvider(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.sqsSettings = sqsSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = new TbAwsSqsAdmin(sqsSettings); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..d9f2cec9f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") +public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueAdmin admin; + + public AwsSqsTbRuleEngineQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsSettings sqsSettings) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.sqsSettings = sqsSettings; + admin = new TbAwsSqsAdmin(sqsSettings); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java new file mode 100644 index 0000000000..01360ebed8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbAwsSqsSettings sqsSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + + public AwsSqsTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbServiceInfoProvider serviceInfoProvider) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.sqsSettings = sqsSettings; + admin = new TbAwsSqsAdmin(sqsSettings); + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + TbAwsSqsProducerTemplate> producerTemplate = + new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + + TbAwsSqsConsumerTemplate> consumerTemplate = + new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java new file mode 100644 index 0000000000..eb4eaed7ed --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.http.SdkHttpMetadata; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.queue.TbQueueMsgMetadata; + +@Data +@AllArgsConstructor +public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { + + private final SdkHttpMetadata metadata; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java new file mode 100644 index 0000000000..6080baec26 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.CreateQueueRequest; +import com.amazonaws.services.sqs.model.QueueAttributeName; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.util.HashMap; +import java.util.Map; + +public class TbAwsSqsAdmin implements TbQueueAdmin { + + private final TbAwsSqsSettings sqsSettings; + private final Map attributes = new HashMap<>(); + private final AWSStaticCredentialsProvider credProvider; + + public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings) { + this.sqsSettings = sqsSettings; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + this.credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + attributes.put("FifoQueue", "true"); + attributes.put("ContentBasedDeduplication", "true"); + attributes.put(QueueAttributeName.VisibilityTimeout.toString(), sqsSettings.getVisibilityTimeout()); + } + + @Override + public void createTopicIfNotExists(String topic) { + AmazonSQS sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + + final CreateQueueRequest createQueueRequest = + new CreateQueueRequest(topic.replaceAll("\\.", "_") + ".fifo") + .withAttributes(attributes); + try { + sqsClient.createQueue(createQueueRequest); + } finally { + if (sqsClient != null) { + sqsClient.shutdown(); + } + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java new file mode 100644 index 0000000000..307e05bcec --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -0,0 +1,239 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; +import com.amazonaws.services.sqs.model.Message; +import com.amazonaws.services.sqs.model.ReceiveMessageRequest; +import com.google.common.reflect.TypeToken; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { + + private static final int MAX_NUM_MSGS = 10; + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final AmazonSQS sqsClient; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbAwsSqsSettings sqsSettings; + + private final List pendingMessages = new CopyOnWriteArrayList<>(); + private volatile Set queueUrls; + private volatile Set partitions; + private ListeningExecutorService consumerExecutor; + private volatile boolean subscribed; + private volatile boolean stopped = false; + + public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.sqsSettings = sqsSettings; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + + if (sqsClient != null) { + sqsClient.shutdown(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); + subscribed = true; + } + + if (!pendingMessages.isEmpty()) { + log.warn("Present {} non committed messages.", pendingMessages.size()); + return Collections.emptyList(); + } + + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .map(msg -> { + try { + return decode(msg); + } catch (IOException e) { + log.error("Failed to decode message: [{}]", msg); + return null; + } + }).filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", topic); + } else { + log.error("Failed to pool messages.", e); + } + } + } + return Collections.emptyList(); + } + + private ListenableFuture> poll(String url, int waitTimeSeconds) { + List>> result = new ArrayList<>(); + + for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { + result.add(consumerExecutor.submit(() -> { + ReceiveMessageRequest request = new ReceiveMessageRequest(); + request + .withWaitTimeSeconds(waitTimeSeconds) + .withMessageAttributeNames("headers") + .withQueueUrl(url) + .withMaxNumberOfMessages(MAX_NUM_MSGS); + return sqsClient.receiveMessage(request).getMessages(); + })); + } + return Futures.transform(Futures.allAsList(result), list -> { + if (!CollectionUtils.isEmpty(list)) { + return list.stream() + .flatMap(messageList -> { + if (!messageList.isEmpty()) { + this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); + return messageList.stream(); + } + return Stream.empty(); + }) + .collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + } + + @Override + public void commit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + + pendingMessages.clear(); + } + + public T decode(Message message) throws InvalidProtocolBufferException { + TbAwsSqsMsg msg = gson.fromJson(message.getBody(), TbAwsSqsMsg.class); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + Map headerMap = gson.fromJson(message.getMessageAttributes().get("headers").getStringValue(), new TypeToken>() { + }.getType()); + headerMap.forEach(headers::put); + msg.setHeaders(headers); + return decoder.decode(msg); + } + + @Data + private static class AwsSqsMsgWrapper { + private final String url; + private final List messages; + + public AwsSqsMsgWrapper(String url, List messages) { + this.url = url; + this.messages = messages; + } + } + + private String getQueueUrl(String topic) { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java new file mode 100644 index 0000000000..4df7558491 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.google.gson.annotations.Expose; +import lombok.Data; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.util.UUID; + +@Data +public class TbAwsSqsMsg implements TbQueueMsg { + private final UUID key; + private final byte[] data; + + public TbAwsSqsMsg(UUID key, byte[] data) { + this.key = key; + this.data = data; + } + + @Expose(serialize = false, deserialize = false) + private TbQueueMsgHeaders headers; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java new file mode 100644 index 0000000000..6514d12747 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -0,0 +1,127 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.MessageAttributeValue; +import com.amazonaws.services.sqs.model.SendMessageRequest; +import com.amazonaws.services.sqs.model.SendMessageResult; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; + +@Slf4j +public class TbAwsSqsProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final AmazonSQS sqsClient; + private final Gson gson = new Gson(); + private final Map queueUrlMap = new ConcurrentHashMap<>(); + private final TbQueueAdmin admin; + private ListeningExecutorService producerExecutor; + + public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + + this.sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credProvider) + .withRegion(sqsSettings.getRegion()) + .build(); + + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + SendMessageRequest sendMsgRequest = new SendMessageRequest(); + sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); + sendMsgRequest.withMessageBody(gson.toJson(new TbAwsSqsMsg(msg.getKey(), msg.getData()))); + + Map attributes = new HashMap<>(); + + attributes.put("headers", new MessageAttributeValue() + .withStringValue(gson.toJson(msg.getHeaders().getData())) + .withDataType("String")); + + sendMsgRequest.withMessageAttributes(attributes); + sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(SendMessageResult result) { + if (callback != null) { + callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata())); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (sqsClient != null) { + sqsClient.shutdown(); + } + } + + private String getQueueUrl(String topic) { + return queueUrlMap.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java new file mode 100644 index 0000000000..42a2f1e983 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") +@Component +@Data +public class TbAwsSqsSettings { + + @Value("${queue.aws_sqs.access_key_id}") + private String accessKeyId; + + @Value("${queue.aws_sqs.secret_access_key}") + private String secretAccessKey; + + @Value("${queue.aws_sqs.region}") + private String region; + + @Value("${queue.aws_sqs.threads_per_topic}") + private int threadsPerTopic; + + @Value("${queue.aws_sqs.visibility_timeout}") + private String visibilityTimeout; +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3891b197a6..3a707d11a7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -138,6 +138,9 @@ public class DefaultTransportService implements TransportService { while (!stopped) { try { List> records = transportNotificationsConsumer.poll(notificationsPollDuration); + if (records.size() == 0) { + continue; + } records.forEach(record -> { try { ToTransportMsg toTransportMsg = record.getValue(); @@ -170,6 +173,10 @@ public class DefaultTransportService implements TransportService { perDeviceLimits.clear(); } stopped = true; + + if (transportNotificationsConsumer != null) { + transportNotificationsConsumer.unsubscribe(); + } if (schedulerExecutor != null) { schedulerExecutor.shutdownNow(); } diff --git a/pom.xml b/pom.xml index 547c97448f..9863982929 100755 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ 2.57 2.7.7 1.23 + 1.11.747 1.5.0 1.4.3 @@ -886,6 +887,11 @@ jts-core ${jts.version} + + com.amazonaws + aws-java-sdk-sqs + ${amazonaws.sqs.version} + org.passay passay diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 373e4107f8..bf629a6f10 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -35,7 +35,7 @@ UTF-8 ${basedir}/../.. - 1.11.323 + 1.11.747 1.83.0 1.16.0 diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index f4a31449e7..6ac9b08777 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or ? + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -71,6 +71,10 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From a6e090ef868fab068c21a65dc76a11c8c240e780 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 31 Mar 2020 16:39:41 +0300 Subject: [PATCH 033/196] Develop/2.5 pubsub (#2566) * created Aws Sqs Queue * improvement AwsSqs providers * created pubsub queue * revert package-lock.json * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * Aws sqs improvements * Created pubsub queue * aws improvements * aws improvements * aws improvements * added visibility timeout to aws queue * pub sub improvements * pub sub improvements * aws sqs improvements * pub sub improvements * added comment to transport.yml about ack deadline --- .../src/main/resources/thingsboard.yml | 10 +- common/queue/pom.xml | 5 +- .../DefaultTbQueueMsg.java} | 7 +- .../common/DefaultTbQueueRequestTemplate.java | 2 +- .../provider/KafkaTbCoreQueueProvider.java | 6 +- .../KafkaTbRuleEngineQueueProvider.java | 10 +- .../provider/PubSubMonolithQueueProvider.java | 137 +++++++++++ .../provider/PubSubTbCoreQueueProvider.java | 113 +++++++++ .../PubSubTbRuleEngineQueueProvider.java | 101 ++++++++ .../PubSubTransportQueueProvider.java | 104 ++++++++ .../server/queue/pubsub/TbPubSubAdmin.java | 157 ++++++++++++ .../pubsub/TbPubSubConsumerTemplate.java | 228 ++++++++++++++++++ .../pubsub/TbPubSubProducerTemplate.java | 135 +++++++++++ .../server/queue/pubsub/TbPubSubSettings.java | 61 +++++ .../queue/sqs/TbAwsSqsConsumerTemplate.java | 3 +- .../queue/sqs/TbAwsSqsProducerTemplate.java | 3 +- pom.xml | 16 +- rule-engine/rule-engine-components/pom.xml | 2 - .../src/main/resources/tb-mqtt-transport.yml | 10 +- 19 files changed, 1078 insertions(+), 32 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/queue/{sqs/TbAwsSqsMsg.java => common/DefaultTbQueueMsg.java} (86%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2fe8f59608..e731bd7bac 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -517,7 +517,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -530,7 +530,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #in seconds + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e201b8995f..0a61eb0cf6 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -56,7 +56,10 @@ com.amazonaws aws-java-sdk-sqs - + + com.google.cloud + google-cloud-pubsub + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java similarity index 86% rename from common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index 4df7558491..0e816ae59b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.sqs; +package org.thingsboard.server.queue.common; import com.google.gson.annotations.Expose; import lombok.Data; @@ -23,16 +23,15 @@ import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.UUID; @Data -public class TbAwsSqsMsg implements TbQueueMsg { +public class DefaultTbQueueMsg implements TbQueueMsg { private final UUID key; private final byte[] data; - public TbAwsSqsMsg(UUID key, byte[] data) { + public DefaultTbQueueMsg(UUID key, byte[] data) { this.key = key; this.data = data; } @Expose(serialize = false, deserialize = false) private TbQueueMsgHeaders headers; - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 40ca2219dd..6ae9b9a33c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate { - log.trace("Received response to Kafka Template request: {}", response); + log.trace("Received response to Queue Template request: {}", response); byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); UUID requestId; if (requestIdHeader == null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java index 177f5f3c3d..110a98b3b9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java @@ -30,7 +30,6 @@ import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -48,21 +47,18 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; public KafkaTbCoreQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportApiSettings transportApiSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java index 36ffd05b48..7f939ee11d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java @@ -27,8 +27,6 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -45,22 +43,16 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; public KafkaTbRuleEngineQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueRuleEngineSettings ruleEngineSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java new file mode 100644 index 0000000000..2dc9549679 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") +public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueAdmin admin; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + private TbQueueProducer> tbCoreProducer; + + public PubSubMonolithQueueProvider(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.admin = new TbPubSubAdmin(pubSubSettings); + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java new file mode 100644 index 0000000000..4c89b35dec --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java @@ -0,0 +1,113 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") +public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueAdmin admin; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + public PubSubTbCoreQueueProvider(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.admin = admin; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToCoreMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getTransportApiRequestConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> getTransportApiResponseProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java new file mode 100644 index 0000000000..3f707235fa --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java @@ -0,0 +1,101 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") +public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueAdmin admin; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + + public PubSubTbRuleEngineQueueProvider(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.admin = admin; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getToRuleEngineMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java new file mode 100644 index 0000000000..b1a7efc950 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class PubSubTransportQueueProvider implements TbTransportQueueProvider { + + private final TbPubSubSettings pubSubSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueAdmin admin; + + public PubSubTransportQueueProvider(TbPubSubSettings pubSubSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.pubSubSettings = pubSubSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.admin = new TbPubSubAdmin(pubSubSettings); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producer); + templateBuilder.responseTemplate(consumer); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> getTransportNotificationsConsumer() { + return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java new file mode 100644 index 0000000000..f0af639d4d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -0,0 +1,157 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.cloud.pubsub.v1.SubscriptionAdminClient; +import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; +import com.google.cloud.pubsub.v1.TopicAdminClient; +import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.pubsub.v1.ListSubscriptionsRequest; +import com.google.pubsub.v1.ListTopicsRequest; +import com.google.pubsub.v1.ProjectName; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PushConfig; +import com.google.pubsub.v1.Subscription; +import com.google.pubsub.v1.Topic; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueAdmin; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +public class TbPubSubAdmin implements TbQueueAdmin { + + private final TbPubSubSettings pubSubSettings; + private final SubscriptionAdminSettings subscriptionAdminSettings; + private final TopicAdminSettings topicAdminSettings; + private final Set topicSet = ConcurrentHashMap.newKeySet(); + private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + + public TbPubSubAdmin(TbPubSubSettings pubSubSettings) { + this.pubSubSettings = pubSubSettings; + + try { + topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create TopicAdminSettings"); + throw new RuntimeException("Failed to create TopicAdminSettings."); + } + + try { + subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + } catch (IOException e) { + log.error("Failed to create SubscriptionAdminSettings"); + throw new RuntimeException("Failed to create SubscriptionAdminSettings."); + } + + try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + topicSet.add(topic.getName()); + } + } catch (IOException e) { + log.error("Failed to get topics.", e); + throw new RuntimeException("Failed to get topics.", e); + } + + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder() + .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) + .build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = + subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + + for (Subscription subscription : response.iterateAll()) { + subscriptionSet.add(subscription.getName()); + } + } catch (IOException e) { + log.error("Failed to get subscriptions.", e); + throw new RuntimeException("Failed to get subscriptions.", e); + } + } + + @Override + public void createTopicIfNotExists(String partition) { + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + + if (topicSet.contains(topicName.toString())) { + createSubscriptionIfNotExists(partition, topicName); + return; + } + + try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; + } + } + + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); + } catch (IOException e) { + log.error("Failed to create topic: [{}].", topicName.toString(), e); + throw new RuntimeException("Failed to create topic.", e); + } + } + + private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + ProjectSubscriptionName subscriptionName = + ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); + + if (subscriptionSet.contains(subscriptionName.toString())) { + return; + } + + try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder() + .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) + .build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = + subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; + } + } + + subscriptionAdminClient.createSubscription( + subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName(); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } catch (IOException e) { + log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e); + throw new RuntimeException("Failed to create subscription.", e); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java new file mode 100644 index 0000000000..4109e72070 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStub; +import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.pubsub.v1.AcknowledgeRequest; +import com.google.pubsub.v1.ProjectSubscriptionName; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.PullRequest; +import com.google.pubsub.v1.PullResponse; +import com.google.pubsub.v1.ReceivedMessage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +public class TbPubSubConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbPubSubSettings pubSubSettings; + + private volatile boolean subscribed; + private volatile Set partitions; + private volatile Set subscriptionNames; + private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); + + private ExecutorService consumerExecutor; + private final SubscriberStub subscriber; + private volatile boolean stopped; + + private volatile int messagesPerTopic; + + public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.pubSubSettings = pubSubSettings; + this.topic = topic; + this.decoder = decoder; + + try { + SubscriberStubSettings subscriberStubSettings = + SubscriberStubSettings.newBuilder() + .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) + .setTransportChannelProvider( + SubscriberStubSettings.defaultGrpcTransportProviderBuilder() + .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) + .build()) + .build(); + + this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); + } catch (IOException e) { + log.error("Failed to create subscriber.", e); + throw new RuntimeException("Failed to create subscriber.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + + if (subscriber != null) { + subscriber.close(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); + subscriptionNames.forEach(admin::createTopicIfNotExists); + consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); + messagesPerTopic = pubSubSettings.getMaxMessages()/subscriptionNames.size(); + subscribed = true; + } + List messages; + try { + messages = receiveMessages(); + if (!messages.isEmpty()) { + List result = new ArrayList<>(); + messages.forEach(msg -> { + try { + result.add(decode(msg.getMessage())); + } catch (InvalidProtocolBufferException e) { + log.error("Failed decode record: [{}]", msg); + } + }); + return result; + } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); + acknowledgeRequests.clear(); + } + + private List receiveMessages() throws ExecutionException, InterruptedException { + List>> result = subscriptionNames.stream().map(subscriptionId -> { + String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); + PullRequest pullRequest = + PullRequest.newBuilder() + .setMaxMessages(messagesPerTopic) + .setReturnImmediately(false) // return immediately if messages are not available + .setSubscription(subscriptionName) + .build(); + + ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); + + return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { + if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { + List ackIds = new ArrayList<>(); + for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { + ackIds.add(message.getAckId()); + } + AcknowledgeRequest acknowledgeRequest = + AcknowledgeRequest.newBuilder() + .setSubscription(subscriptionName) + .addAllAckIds(ackIds) + .build(); + + acknowledgeRequests.add(acknowledgeRequest); + return pullResponse.getReceivedMessagesList(); + } + return null; + }, consumerExecutor); + + }).collect(Collectors.toList()); + + ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { + if (!CollectionUtils.isEmpty(listMessages)) { + return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); + } + return Collections.emptyList(); + }, consumerExecutor); + + return transform.get(); + } + + public T decode(PubsubMessage message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); + TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + Map headerMap = gson.fromJson(message.getAttributesMap().get("headers"), new TypeToken>() { + }.getType()); + headerMap.forEach(headers::put); + msg.setHeaders(headers); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java new file mode 100644 index 0000000000..fb9bc8b189 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.gson.Gson; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.ProjectTopicName; +import com.google.pubsub.v1.PubsubMessage; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class TbPubSubProducerTemplate implements TbQueueProducer { + + private final Gson gson = new Gson(); + + private final String defaultTopic; + private final TbQueueAdmin admin; + private final TbPubSubSettings pubSubSettings; + + private final Map publisherMap = new ConcurrentHashMap<>(); + + private ExecutorService pubExecutor = Executors.newCachedThreadPool(); + + public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { + this.defaultTopic = defaultTopic; + this.admin = admin; + this.pubSubSettings = pubSubSettings; + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); + pubsubMessageBuilder.setData(getMsg(msg)); + pubsubMessageBuilder.putAttributes("headers", gson.toJson(msg.getHeaders().getData())); + + Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); + ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); + + ApiFutures.addCallback(future, new ApiFutureCallback() { + public void onSuccess(String messageId) { + if (callback != null) { + callback.onSuccess(null); + } + } + + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }, pubExecutor); + } + + @Override + public void stop() { + publisherMap.forEach((k, v) -> { + if (v != null) { + try { + v.shutdown(); + v.awaitTermination(1, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to shutdown PubSub client during destroy()", e); + } + } + }); + + if (pubExecutor != null) { + pubExecutor.shutdownNow(); + } + } + + private ByteString getMsg(T msg) { + String json = gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData())); + return ByteString.copyFrom(json.getBytes()); + } + + private Publisher getOrCreatePublisher(String topic) { + if (publisherMap.containsKey(topic)) { + return publisherMap.get(topic); + } else { + try { + admin.createTopicIfNotExists(topic); + ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); + Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + publisherMap.put(topic, publisher); + return publisher; + } catch (IOException e) { + log.error("Failed to create topic [{}].", topic, e); + throw new RuntimeException("Failed to create topic.", e); + } + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java new file mode 100644 index 0000000000..851a2b3245 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.ByteArrayInputStream; +import java.io.IOException; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +@Component +@Data +public class TbPubSubSettings { + + @Value("${queue.pubsub.project_id}") + private String projectId; + + @Value("${queue.pubsub.service_account}") + private String serviceAccount; + + @Value("${queue.pubsub.ack_deadline}") + private int ackDeadline; + + @Value("${queue.pubsub.max_msg_size}") + private int maxMsgSize; + + @Value("${queue.pubsub.max_messages}") + private int maxMessages; + + private CredentialsProvider credentialsProvider; + + @PostConstruct + private void init() throws IOException { + ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( + new ByteArrayInputStream(serviceAccount.getBytes())); + credentialsProvider = FixedCredentialsProvider.create(credentials); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 307e05bcec..b4a2f84b83 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -39,6 +39,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.io.IOException; @@ -212,7 +213,7 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo } public T decode(Message message) throws InvalidProtocolBufferException { - TbAwsSqsMsg msg = gson.fromJson(message.getBody(), TbAwsSqsMsg.class); + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); Map headerMap = gson.fromJson(message.getMessageAttributes().get("headers").getStringValue(), new TypeToken>() { }.getType()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 6514d12747..a5707c845d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -35,6 +35,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.HashMap; import java.util.Map; @@ -79,7 +80,7 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { SendMessageRequest sendMsgRequest = new SendMessageRequest(); sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new TbAwsSqsMsg(msg.getKey(), msg.getData()))); + sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData()))); Map attributes = new HashMap<>(); diff --git a/pom.xml b/pom.xml index 9863982929..e385bff9ce 100755 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 1.4.3 4.2.0 3.5.5 - 3.6.1 + 3.9.1 1.22.1 1.16.18 1.1.0 @@ -93,6 +93,7 @@ 2.7.7 1.23 1.11.747 + 1.84.0 1.5.0 1.4.3 @@ -888,10 +889,15 @@ ${jts.version} - com.amazonaws - aws-java-sdk-sqs - ${amazonaws.sqs.version} - + com.amazonaws + aws-java-sdk-sqs + ${amazonaws.sqs.version} + + + com.google.cloud + google-cloud-pubsub + ${pubsub.client.version} + org.passay passay diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index bf629a6f10..645419ca41 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -36,7 +36,6 @@ UTF-8 ${basedir}/../.. 1.11.747 - 1.83.0 1.16.0 @@ -99,7 +98,6 @@ com.google.cloud google-cloud-pubsub - ${pubsub.client.version} com.google.api.grpc diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 6ac9b08777..a875d56782 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -75,6 +75,14 @@ queue: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From b39328c989a36711c133de67d753cbdedc2a4467 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 31 Mar 2020 16:47:43 +0300 Subject: [PATCH 034/196] [2.5]fix ConcurrentModificationException (#2560) * fix ConcurrentModificationException * kafka consumer improvements * kafka consumer improvements * refactored kafka consumer * refactored kafka consumer --- .../queue/kafka/TBKafkaConsumerTemplate.java | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index 68b9cea7fa..d882153177 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -33,6 +33,8 @@ import java.util.Collections; import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -46,6 +48,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final TbKafkaDecoder decoder; private volatile boolean subscribed; private volatile Set partitions; + private final Lock consumerLock; @Getter private final String topic; @@ -71,6 +74,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.topic = topic; + this.consumerLock = new ReentrantLock(); } @Override @@ -94,23 +98,30 @@ public class TBKafkaConsumerTemplate implements TbQueueCon log.debug("Failed to await subscription", e); } } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; + try { + consumerLock.lock(); + + if (!subscribed) { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); + subscribed = true; + } + + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.count() > 0) { + List result = new ArrayList<>(); + records.forEach(record -> { + try { + result.add(decode(record)); + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + } + }); + return result; + } + } finally { + consumerLock.unlock(); } } return Collections.emptyList(); @@ -118,12 +129,25 @@ public class TBKafkaConsumerTemplate implements TbQueueCon @Override public void commit() { - consumer.commitAsync(); + try { + consumerLock.lock(); + consumer.commitAsync(); + } finally { + consumerLock.unlock(); + } } @Override public void unsubscribe() { - consumer.unsubscribe(); + try { + consumerLock.lock(); + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); + } + } finally { + consumerLock.unlock(); + } } public T decode(ConsumerRecord record) throws IOException { From d94cb9ca4763ffadde89886966121ffa2b58d7a2 Mon Sep 17 00:00:00 2001 From: VoBa Date: Wed, 1 Apr 2020 09:33:46 +0300 Subject: [PATCH 035/196] Added correct ts on alarm clear and ack actions (#2550) --- .../thingsboard/server/controller/AlarmController.java | 8 ++++++-- .../org/thingsboard/server/dao/alarm/AlarmService.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 09e6dd5d10..140c9ecf86 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -118,7 +118,9 @@ public class AlarmController extends BaseController { try { AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.WRITE); - alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, System.currentTimeMillis()).get(); + long ackTs = System.currentTimeMillis(); + alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get(); + alarm.setAckTs(ackTs); logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null); } catch (Exception e) { throw handleException(e); @@ -133,7 +135,9 @@ public class AlarmController extends BaseController { try { AlarmId alarmId = new AlarmId(toUUID(strAlarmId)); Alarm alarm = checkAlarmId(alarmId, Operation.WRITE); - alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, System.currentTimeMillis()).get(); + long clearTs = System.currentTimeMillis(); + alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get(); + alarm.setClearTs(clearTs); logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null); } catch (Exception e) { throw handleException(e); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 19cf4bbbb3..e2277497a5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -39,7 +39,7 @@ public interface AlarmService { ListenableFuture ackAlarm(TenantId tenantId, AlarmId alarmId, long ackTs); - ListenableFuture clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long ackTs); + ListenableFuture clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long clearTs); ListenableFuture findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId); From 00b5d36e0bf8d8295463018537dbbd5744902b7b Mon Sep 17 00:00:00 2001 From: blackstar-baba <535650957@qq.com> Date: Wed, 1 Apr 2020 14:39:51 +0800 Subject: [PATCH 036/196] fix bug: Under high concurrency, Mqtt client nextMessageId exceeds the 0xffff limit (#2564) The compareAndSet method and getAndIncrement method of the AtomicInteger class are atomic, but when these two methods are used at the same time, they are no longer atomic. --- .../main/java/org/thingsboard/mqtt/MqttClientImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java index 169779b530..aef2cad684 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java @@ -407,8 +407,12 @@ final class MqttClientImpl implements MqttClient { } private MqttMessageIdVariableHeader getNewMessageId() { - this.nextMessageId.compareAndSet(0xffff, 1); - return MqttMessageIdVariableHeader.from(this.nextMessageId.getAndIncrement()); + int messageId; + synchronized (this.nextMessageId) { + this.nextMessageId.compareAndSet(0xffff, 1); + messageId = this.nextMessageId.getAndIncrement(); + } + return MqttMessageIdVariableHeader.from(messageId); } private Future createSubscription(String topic, MqttHandler handler, boolean once, MqttQoS qos) { From f63864f06ecfc4a871ad6cfc754c7e452133b2a3 Mon Sep 17 00:00:00 2001 From: VoBa Date: Wed, 1 Apr 2020 09:41:23 +0300 Subject: [PATCH 037/196] Filter relation by To and From entities (#2567) * Filter relation by To and From entities * Rename typo --- .../controller/EntityRelationController.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index ea87f86781..8058cca685 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -24,13 +24,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; @@ -38,6 +36,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; +import java.util.stream.Collectors; @RestController @@ -167,7 +166,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByFrom(getTenantId(), entityId, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByFrom(getTenantId(), entityId, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -185,7 +184,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get())); } catch (Exception e) { throw handleException(e); } @@ -205,7 +204,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -223,7 +222,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByTo(getTenantId(), entityId, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByTo(getTenantId(), entityId, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -241,7 +240,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get())); } catch (Exception e) { throw handleException(e); } @@ -261,7 +260,7 @@ public class EntityRelationController extends BaseController { checkEntityId(entityId, Operation.READ); RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); try { - return checkNotNull(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup)); + return checkNotNull(filterRelationsByReadPermission(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup))); } catch (Exception e) { throw handleException(e); } @@ -276,7 +275,7 @@ public class EntityRelationController extends BaseController { checkNotNull(query.getFilters()); checkEntityId(query.getParameters().getEntityId(), Operation.READ); try { - return checkNotNull(relationService.findByQuery(getTenantId(), query).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findByQuery(getTenantId(), query).get())); } catch (Exception e) { throw handleException(e); } @@ -291,12 +290,28 @@ public class EntityRelationController extends BaseController { checkNotNull(query.getFilters()); checkEntityId(query.getParameters().getEntityId(), Operation.READ); try { - return checkNotNull(relationService.findInfoByQuery(getTenantId(), query).get()); + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByQuery(getTenantId(), query).get())); } catch (Exception e) { throw handleException(e); } } + private List filterRelationsByReadPermission(List relationsByQuery) { + return relationsByQuery.stream().filter(relationByQuery -> { + try { + checkEntityId(relationByQuery.getTo(), Operation.READ); + } catch (ThingsboardException e) { + return false; + } + try { + checkEntityId(relationByQuery.getFrom(), Operation.READ); + } catch (ThingsboardException e) { + return false; + } + return true; + }).collect(Collectors.toList()); + } + private RelationTypeGroup parseRelationTypeGroup(String strRelationTypeGroup, RelationTypeGroup defaultValue) { RelationTypeGroup result = defaultValue; if (strRelationTypeGroup != null && strRelationTypeGroup.trim().length() > 0) { From fef74373e0a5b5ff95308ac21fe52c68f74036a5 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Wed, 1 Apr 2020 09:42:03 +0300 Subject: [PATCH 038/196] Fix timeseries table layout (#2569) --- .../widget/lib/timeseries-table-widget.scss | 4 - .../lib/timeseries-table-widget.tpl.html | 96 ++++++++++--------- 2 files changed, 49 insertions(+), 51 deletions(-) diff --git a/ui/src/app/widget/lib/timeseries-table-widget.scss b/ui/src/app/widget/lib/timeseries-table-widget.scss index 4381ee7f1e..9e8ed9ea3a 100644 --- a/ui/src/app/widget/lib/timeseries-table-widget.scss +++ b/ui/src/app/widget/lib/timeseries-table-widget.scss @@ -31,10 +31,6 @@ tb-timeseries-table-widget { z-index: 10; } - md-table-container { - overflow-x: visible; - } - md-tabs:not(.md-no-tab-content):not(.md-dynamic-height) { min-height: 0; } diff --git a/ui/src/app/widget/lib/timeseries-table-widget.tpl.html b/ui/src/app/widget/lib/timeseries-table-widget.tpl.html index b41bdf7d5c..1a7194b3f9 100644 --- a/ui/src/app/widget/lib/timeseries-table-widget.tpl.html +++ b/ui/src/app/widget/lib/timeseries-table-widget.tpl.html @@ -38,56 +38,58 @@ - - - - - - - - - +
+ +
- Timestamp - - {{ h.dataKey.label }} -
+ + + + + + - - - - - - -
+ Timestamp + + {{ h.dataKey.label }} +
- - {{actionDescriptor.icon}} - - {{ actionDescriptor.displayName }} - - -
- - widget.no-data-found - + + + + + + {{actionDescriptor.icon}} + + {{ actionDescriptor.displayName }} + + + + + + + + widget.no-data-found + + Date: Wed, 1 Apr 2020 15:43:06 +0900 Subject: [PATCH 039/196] tools/src/main/shell/*.sh: Fix "yes" patterns (#2553) So that it accepts "yes". --- tools/src/main/shell/client.keygen.sh | 2 +- tools/src/main/shell/server.keygen.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/src/main/shell/client.keygen.sh b/tools/src/main/shell/client.keygen.sh index d4e750dc87..c3fd45c186 100755 --- a/tools/src/main/shell/client.keygen.sh +++ b/tools/src/main/shell/client.keygen.sh @@ -55,7 +55,7 @@ while : echo "Done" exit 0 ;; - [yY]|[yY][eE]|[yY][eE]|[sS]|[yY]|"") + [yY]|[yY][eE]|[yY][eE][sS]|"") echo "Cleaning up files" rm -rf $CLIENT_FILE_PREFIX.jks rm -rf $CLIENT_FILE_PREFIX.pub.pem diff --git a/tools/src/main/shell/server.keygen.sh b/tools/src/main/shell/server.keygen.sh index c3a7dbbca3..c01d3ead50 100755 --- a/tools/src/main/shell/server.keygen.sh +++ b/tools/src/main/shell/server.keygen.sh @@ -71,7 +71,7 @@ while : echo "Done" exit 0 ;; - [yY]|[yY][eE]|[yY][eE]|[sS]|[yY]|"") + [yY]|[yY][eE]|[yY][eE][sS]|"") echo "Cleaning up files" rm -rf $SERVER_FILE_PREFIX.jks rm -rf $SERVER_FILE_PREFIX.pub.pem @@ -129,7 +129,7 @@ if [[ $COPY = true ]]; then [nN]|[nN][oO]) break ;; - [yY]|[yY][eE]|[yY][eE]|[sS]|[yY]|"") + [yY]|[yY][eE]|[yY][eE][sS]|"") read -p "(Default: $SERVER_KEYSTORE_DIR): " dir if [[ ! -z $dir ]]; then DESTINATION=$dir; From 638ca0e1d104d8ce2ce744e284a777abb0b61bca Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Apr 2020 18:35:43 +0300 Subject: [PATCH 040/196] Refactoring to multiple queues for rule engine --- .../server/actors/ActorSystemContext.java | 13 +- .../actors/ruleChain/DefaultTbContext.java | 167 ++++++++++++------ .../RuleChainActorMessageProcessor.java | 4 +- .../RuleNodeActorMessageProcessor.java | 2 +- .../actors/service/DefaultActorService.java | 12 +- .../server/actors/tenant/TenantActor.java | 2 +- .../server/controller/BaseController.java | 4 +- .../controller/RuleChainController.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 41 +++-- .../DefaultTbRuleEngineConsumerService.java | 84 +++++++-- .../processing/AbstractConsumerService.java | 37 ++-- ...TbRuleEngineProcessingStrategyFactory.java | 34 ++-- .../rpc/DefaultTbCoreDeviceRpcService.java | 4 +- .../script/RuleNodeJsScriptEngine.java | 2 +- .../state/DefaultDeviceStateService.java | 8 +- .../DefaultTbLocalSubscriptionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../transport/TbCoreTransportApiService.java | 12 +- application/src/main/resources/logback.xml | 1 - .../src/main/resources/thingsboard.yml | 46 +++-- ...AbstractRuleEngineFlowIntegrationTest.java | 9 +- ...actRuleEngineLifecycleIntegrationTest.java | 5 +- .../script/RuleNodeJsScriptEngineTest.java | 14 +- .../thingsboard/server/common/msg/TbMsg.java | 30 +++- .../common/msg/queue/PartitionChangeMsg.java | 2 +- .../msg/queue/QueueToRuleEngineMsg.java | 6 + .../server/common/msg/queue/ServiceQueue.java | 62 +++++++ .../{ServiceKey.java => ServiceQueueKey.java} | 19 +- .../server/common/msg/queue/ServiceType.java | 1 + .../MultipleTbQueueTbMsgCallbackWrapper.java | 4 +- .../TbQueueTbMsgCallbackWrapper.java | 4 +- .../discovery/ClusterTopologyChangeEvent.java | 8 +- .../ConsistentHashPartitionService.java | 154 +++++++++------- .../DefaultTbServiceInfoProvider.java | 22 ++- .../queue/discovery/PartitionChangeEvent.java | 13 +- .../queue/discovery/PartitionService.java | 4 +- .../discovery/TbServiceInfoProvider.java | 5 - .../discovery/TopicPartitionInfoKey.java | 8 +- ...r.java => AwsSqsMonolithQueueFactory.java} | 45 ++--- ...der.java => AwsSqsTbCoreQueueFactory.java} | 38 ++-- ...va => AwsSqsTbRuleEngineQueueFactory.java} | 29 +-- ....java => AwsSqsTransportQueueFactory.java} | 22 +-- ...java => InMemoryMonolithQueueFactory.java} | 41 ++--- ...a => InMemoryTbTransportQueueFactory.java} | 26 +-- ...er.java => KafkaMonolithQueueFactory.java} | 50 +++--- ...ider.java => KafkaTbCoreQueueFactory.java} | 36 ++-- ...ava => KafkaTbRuleEngineQueueFactory.java} | 34 ++-- ...java => KafkaTbTransportQueueFactory.java} | 30 ++-- ...r.java => PubSubMonolithQueueFactory.java} | 47 ++--- ...der.java => PubSubTbCoreQueueFactory.java} | 36 ++-- ...va => PubSubTbRuleEngineQueueFactory.java} | 33 ++-- ....java => PubSubTransportQueueFactory.java} | 30 ++-- ...eProvider.java => TbCoreQueueFactory.java} | 20 +-- .../provider/TbCoreQueueProducerProvider.java | 14 +- .../TbRuleEngineProducerProvider.java | 14 +- ...der.java => TbRuleEngineQueueFactory.java} | 19 +- ...ider.java => TbTransportQueueFactory.java} | 10 +- .../TbTransportQueueProducerProvider.java | 8 +- .../{ => settings}/TbQueueCoreSettings.java | 2 +- .../settings/TbQueueRuleEngineSettings.java | 45 +++++ .../TbQueueTransportApiSettings.java | 2 +- .../TbQueueTransportNotificationSettings.java | 2 +- ...leEngineQueueAckStrategyConfiguration.java | 32 ++++ .../TbRuleEngineQueueConfiguration.java} | 15 +- common/queue/src/main/proto/queue.proto | 8 + .../service/DefaultTransportService.java | 18 +- .../rule/engine/api/TbContext.java | 79 ++++++++- .../engine/action/TbAbstractAlarmNode.java | 10 +- .../action/TbAbstractCustomerActionNode.java | 4 +- .../action/TbAbstractRelationActionNode.java | 12 +- .../rule/engine/action/TbMsgCountNode.java | 7 +- .../rule/engine/delay/TbMsgDelayNode.java | 2 +- .../engine/geo/TbGpsGeofencingActionNode.java | 4 +- .../rule/engine/kafka/TbKafkaNode.java | 3 +- .../engine/rest/TbRedisQueueProcessor.java | 2 +- .../TbSynchronizationBeginNode.java | 3 +- .../rule/engine/action/TbAlarmNodeTest.java | 10 +- .../engine/filter/TbJsFilterNodeTest.java | 6 +- .../engine/filter/TbJsSwitchNodeTest.java | 2 +- .../engine/mail/TbMsgToEmailNodeTest.java | 2 +- .../TbGetCustomerAttributeNodeTest.java | 16 +- .../transform/TbChangeOriginatorNodeTest.java | 6 +- .../transform/TbTransformMsgNodeTest.java | 6 +- 83 files changed, 1066 insertions(+), 665 deletions(-) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java rename common/message/src/main/java/org/thingsboard/server/common/msg/queue/{ServiceKey.java => ServiceQueueKey.java} (72%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => common}/MultipleTbQueueTbMsgCallbackWrapper.java (90%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => common}/TbQueueTbMsgCallbackWrapper.java (88%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsMonolithQueueProvider.java => AwsSqsMonolithQueueFactory.java} (76%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsTbCoreQueueProvider.java => AwsSqsTbCoreQueueFactory.java} (76%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsTbRuleEngineQueueProvider.java => AwsSqsTbRuleEngineQueueFactory.java} (74%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{AwsSqsTransportQueueProvider.java => AwsSqsTransportQueueFactory.java} (85%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{InMemoryMonolithQueueProvider.java => InMemoryMonolithQueueFactory.java} (70%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{InMemoryTbTransportQueueProvider.java => InMemoryTbTransportQueueFactory.java} (77%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaMonolithQueueProvider.java => KafkaMonolithQueueFactory.java} (82%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTbCoreQueueProvider.java => KafkaTbCoreQueueFactory.java} (85%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTbRuleEngineQueueProvider.java => KafkaTbRuleEngineQueueFactory.java} (80%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{KafkaTbTransportQueueProvider.java => KafkaTbTransportQueueFactory.java} (83%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubMonolithQueueProvider.java => PubSubMonolithQueueFactory.java} (75%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubTbCoreQueueProvider.java => PubSubTbCoreQueueFactory.java} (78%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubTbRuleEngineQueueProvider.java => PubSubTbRuleEngineQueueFactory.java} (74%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{PubSubTransportQueueProvider.java => PubSubTransportQueueFactory.java} (79%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TbCoreQueueProvider.java => TbCoreQueueFactory.java} (75%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TbRuleEngineQueueProvider.java => TbRuleEngineQueueFactory.java} (75%) rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{TbTransportQueueProvider.java => TbTransportQueueFactory.java} (78%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => settings}/TbQueueCoreSettings.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java rename common/queue/src/main/java/org/thingsboard/server/queue/{ => settings}/TbQueueTransportApiSettings.java (96%) rename common/queue/src/main/java/org/thingsboard/server/queue/{ => settings}/TbQueueTransportNotificationSettings.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java rename common/queue/src/main/java/org/thingsboard/server/queue/{TbQueueRuleEngineSettings.java => settings/TbRuleEngineQueueConfiguration.java} (71%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index bd7c25ddc9..696383b242 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -31,7 +31,6 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -48,7 +47,6 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.auth.DeviceAuthService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -413,7 +411,12 @@ public class ActorSystemContext { return partitionService.resolve(serviceType, tenantId, entityId); } - public String getServerAddress() { + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return partitionService.resolve(serviceType, queueName, tenantId, entityId); + } + + + public String getServiceId() { return serviceInfoProvider.getServiceId(); } @@ -445,7 +448,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() .put("type", type) - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("entityId", tbMsg.getOriginator().getId().toString()) .put("entityName", tbMsg.getOriginator().getEntityType().name()) .put("msgId", tbMsg.getId().toString()) @@ -505,7 +508,7 @@ public class ActorSystemContext { ObjectNode node = mapper.createObjectNode() //todo: what fields are needed here? - .put("server", getServerAddress()) + .put("server", getServiceId()) .put("message", "Reached debug mode rate limit!"); if (error != null) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 7c831929ca..2f31d1dc35 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -22,12 +22,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; -import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -37,18 +36,17 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -63,10 +61,11 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -78,6 +77,7 @@ import java.util.function.Consumer; /** * Created by ashvayka on 19.03.18. */ +@Slf4j class DefaultTbContext implements TbContext { public final static ObjectMapper mapper = new ObjectMapper(); @@ -100,11 +100,6 @@ class DefaultTbContext implements TbContext { tellNext(msg, relationTypes, null); } - @Override - public void tellNext(TbMsg msg, String relationType, Throwable th) { - tellNext(msg, Collections.singleton(relationType), th); - } - private void tellNext(TbMsg msg, Set relationTypes, Throwable th) { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); @@ -118,6 +113,77 @@ class DefaultTbContext implements TbContext { scheduleMsgWithDelay(new RuleNodeToSelfMsg(msg), delayMs, nodeCtx.getSelfActor()); } + @Override + public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + @Override + public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueue(tpi, tbMsg, onFailure, onSuccess); + } + + private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer onFailure, Runnable onSuccess) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + } + + @Override + public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + } + + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)) + .addAllRelationTypes(relationTypes) + .build(); + mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + } + + @Override + public void ack(TbMsg tbMsg) { + tbMsg.getCallback().onSuccess(); + } + @Override public boolean isLocalEntity(EntityId entityId) { return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); @@ -135,68 +201,44 @@ class DefaultTbContext implements TbContext { nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); } - @Override public void updateSelf(RuleNode self) { nodeCtx.setSelf(self); } @Override public TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(UUIDs.timeBased(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId(), null); + return TbMsg.newMsg(type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); } @Override public TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { - return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), null); - } - - @Override - public void sendTbMsgToRuleEngine(TbMsg tbMsg) { - mainCtx.getClusterService().onToRuleEngineMsg(getTenantId(), tbMsg.getOriginator(), tbMsg); + return TbMsg.transformMsg(origMsg, type, originator, metaData, data); } public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(customer); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, customer.getId(), - getActionMetaData(ruleNodeId), TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process customer created msg: " + e); - } + return entityCreatedMsg(customer, customer.getId(), ruleNodeId); } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(device); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, device.getId(), getActionMetaData(ruleNodeId), - TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process device created msg: " + e); - } + return entityCreatedMsg(device, device.getId(), ruleNodeId); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { - try { - ObjectNode entityNode = mapper.valueToTree(asset); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, asset.getId(), getActionMetaData(ruleNodeId), - TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); - } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process asset created msg: " + e); - } + return entityCreatedMsg(asset, asset.getId(), ruleNodeId); } public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) { + return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); + } + + public TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { try { - ObjectNode entityNode = mapper.valueToTree(alarm); - return new TbMsg(UUIDs.timeBased(), DataConstants.ENTITY_CREATED, alarm.getId(), getActionMetaData(ruleNodeId), - TbMsgDataType.JSON, mapper.writeValueAsString(entityNode), null, null, null); + return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); } catch (JsonProcessingException | IllegalArgumentException e) { - throw new RuntimeException("Failed to process alarm created msg: " + e); + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); } } - @Override public RuleNodeId getSelfId() { return nodeCtx.getSelf().getId(); @@ -254,7 +296,7 @@ class DefaultTbContext implements TbContext { } @Override - public String getNodeId() { + public String getServiceId() { return mainCtx.getServiceInfoProvider().getServiceId(); } @@ -362,10 +404,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getRedisTemplate(); } - @Override - public String getServerAddress() { - return mainCtx.getServerAddress(); - } private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); @@ -373,4 +411,29 @@ class DefaultTbContext implements TbContext { return metaData; } + private class SimpleTbQueueCallback implements TbQueueCallback { + private final Runnable onSuccess; + private final Consumer onFailure; + + public SimpleTbQueueCallback(Runnable onSuccess, Consumer onFailure) { + this.onSuccess = onSuccess; + this.onFailure = onFailure; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (onSuccess != null) { + onSuccess.run(); + } + } + + @Override + public void onFailure(Throwable t) { + if (onFailure != null) { + onFailure.accept(t); + } else { + log.debug("[{}] Failed to put item into queue", nodeCtx.getTenantId(), t); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 50aa7e5dd6..00d9d85b1c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -39,10 +39,10 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.MultipleTbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index 2dd6764407..4cd668cc36 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -40,7 +40,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor implements TbCoreConsumerService { +public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { - @Value("${queue.core.poll_interval}") + @Value("${queue.core.poll-interval}") private long pollDuration; - @Value("${queue.core.pack_processing_timeout}") + @Value("${queue.core.pack-processing-timeout}") private long packProcessingTimeout; @Value("${queue.core.stats.enabled:false}") private boolean statsEnabled; + private final TbQueueConsumer> mainConsumer; private final DeviceStateService stateService; private final TbLocalSubscriptionService localSubscriptionService; private final SubscriptionManagerService subscriptionManagerService; private final TbCoreDeviceRpcService tbCoreDeviceRpcService; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); - public DefaultTbCoreConsumerService(TbCoreQueueProvider tbCoreQueueProvider, ActorSystemContext actorContext, + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService, SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService, TbCoreDeviceRpcService tbCoreDeviceRpcService) { - super(actorContext, encodingService, - tbCoreQueueProvider.getToCoreMsgConsumer(), tbCoreQueueProvider.getToCoreNotificationsMsgConsumer()); + super(actorContext, encodingService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); + this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); this.stateService = stateService; this.localSubscriptionService = localSubscriptionService; this.subscriptionManagerService = subscriptionManagerService; @@ -96,8 +99,16 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); + this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumersExecutor.submit(() -> { while (!stopped) { try { List> msgs = mainConsumer.poll(pollDuration); @@ -171,7 +182,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbMsgCallback callback) throws Exception { + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) { ToCoreNotificationMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); @@ -197,7 +208,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { +public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - @Value("${queue.rule_engine.poll_interval}") + @Value("${queue.rule-engine.poll-interval}") private long pollDuration; - @Value("${queue.rule_engine.pack_processing_timeout}") + @Value("${queue.rule-engine.pack-processing-timeout}") private long packProcessingTimeout; - @Value("${queue.rule_engine.stats.enabled:false}") + @Value("${queue.rule-engine.stats.enabled:false}") private boolean statsEnabled; private final TbCoreConsumerStats stats = new TbCoreConsumerStats(); private final TbRuleEngineProcessingStrategyFactory factory; + private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbRuleEngineQueueProvider tbRuleEngineQueueProvider, + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbQueueRuleEngineSettings ruleEngineSettings, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { - super(actorContext, encodingService, - tbRuleEngineQueueProvider.getToRuleEngineMsgConsumer(), tbRuleEngineQueueProvider.getToRuleEngineNotificationsMsgConsumer()); + super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.ruleEngineSettings = ruleEngineSettings; + this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.factory = factory; } @PostConstruct public void init() { super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); - this.factory.newInstance(); + for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + } + } + + @Override + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (partitionChangeEvent.getServiceType().equals(getServiceType())) { + ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); + log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); + consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); + } + } + + @Override + protected void launchMainConsumers() { + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue))); } @Override - protected void launchMainConsumer() { - mainConsumerExecutor.execute(() -> { + protected void stopMainConsumers() { + consumers.values().forEach(TbQueueConsumer::unsubscribe); + } + + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration) { + consumersExecutor.execute(() -> { while (!stopped) { try { - List> msgs = mainConsumer.poll(pollDuration); + List> msgs = consumer.poll(pollDuration); if (msgs.isEmpty()) { continue; } - TbRuleEngineProcessingStrategy strategy = factory.newInstance(); + TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getAckStrategy()); TbRuleEngineProcessingDecision decision = null; boolean firstAttempt = true; while (!stopped && (firstAttempt || !decision.isCommit())) { @@ -111,7 +146,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(tenantId, toRuleEngineMsg.getTbMsg(), callback); + forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); } else { callback.onSuccess(); } @@ -126,7 +161,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); } - mainConsumer.commit(); + consumer.commit(); } catch (Exception e) { if (!stopped) { log.warn("Failed to process messages from queue.", e); @@ -172,16 +207,27 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } - private void forwardToRuleEngineActor(TenantId tenantId, ByteString tbMsgData, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(tbMsgData.toByteArray(), callback); - actorContext.getAppActor().tell(new QueueToRuleEngineMsg(tenantId, tbMsg), ActorRef.noSender()); + private void forwardToRuleEngineActor(TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes = null; + if (relationTypesList != null) { + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes); + actorContext.getAppActor().tell(msg, ActorRef.noSender()); //TODO: 2.5 before release. // if (statsEnabled) { // stats.log(toDeviceActorMsg); // } } - @Scheduled(fixedDelayString = "${queue.rule_engine.stats.print_interval_ms}") + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { stats.printStats(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 781762a352..0df0517232 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -42,55 +42,48 @@ import java.util.function.Function; import java.util.stream.Collectors; @Slf4j -public abstract class AbstractConsumerService implements ApplicationListener { +public abstract class AbstractConsumerService implements ApplicationListener { - protected volatile ExecutorService mainConsumerExecutor; - private volatile ExecutorService notificationsConsumerExecutor; + protected volatile ExecutorService consumersExecutor; + protected volatile ExecutorService notificationsConsumerExecutor; protected volatile boolean stopped = false; protected final ActorSystemContext actorContext; protected final DataDecodingEncodingService encodingService; - protected final TbQueueConsumer> mainConsumer; + protected final TbQueueConsumer> nfConsumer; - public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> mainConsumer, TbQueueConsumer> nfConsumer) { + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbQueueConsumer> nfConsumer) { this.actorContext = actorContext; this.encodingService = encodingService; - this.mainConsumer = mainConsumer; this.nfConsumer = nfConsumer; } public void init(String mainConsumerThreadName, String nfConsumerThreadName) { - this.mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName(mainConsumerThreadName)); this.notificationsConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(nfConsumerThreadName)); } - @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceKey().getServiceType() == getServiceType()) { - log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); - this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); - } - } - @EventListener(ApplicationReadyEvent.class) public void onApplicationEvent(ApplicationReadyEvent event) { log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); this.nfConsumer.subscribe(); launchNotificationsConsumer(); - launchMainConsumer(); + launchMainConsumers(); } protected abstract ServiceType getServiceType(); - protected abstract void launchMainConsumer(); + protected abstract void launchMainConsumers(); + + protected abstract void stopMainConsumers(); protected abstract long getNotificationPollDuration(); protected abstract long getNotificationPackProcessingTimeout(); protected void launchNotificationsConsumer() { - notificationsConsumerExecutor.execute(() -> { + notificationsConsumerExecutor.submit(() -> { while (!stopped) { try { List> msgs = nfConsumer.poll(getNotificationPollDuration()); @@ -137,16 +130,14 @@ public abstract class AbstractConsumerService ServiceType.TB_CORE.equals(key.getServiceType()))) { + if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { /* * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 0397b8cc85..77b0967966 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -113,7 +113,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio @Override @EventListener(PartitionChangeEvent.class) public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceKey().getServiceType())) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { currentPartitions.clear(); currentPartitions.addAll(partitionChangeEvent.getPartitions()); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 7a5735b843..566bf99cd3 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.provider.TbCoreQueueProvider; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; @@ -42,7 +42,7 @@ import java.util.concurrent.*; @TbCoreComponent public class TbCoreTransportApiService { - private final TbCoreQueueProvider tbCoreQueueProvider; + private final TbCoreQueueFactory tbCoreQueueFactory; private final TransportApiService transportApiService; @Value("${queue.transport_api.max_pending_requests:10000}") @@ -58,16 +58,16 @@ public class TbCoreTransportApiService { private TbQueueResponseTemplate, TbProtoQueueMsg> transportApiTemplate; - public TbCoreTransportApiService(TbCoreQueueProvider tbCoreQueueProvider, TransportApiService transportApiService) { - this.tbCoreQueueProvider = tbCoreQueueProvider; + public TbCoreTransportApiService(TbCoreQueueFactory tbCoreQueueFactory, TransportApiService transportApiService) { + this.tbCoreQueueFactory = tbCoreQueueFactory; this.transportApiService = transportApiService; } @PostConstruct public void init() { this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); - TbQueueProducer> producer = tbCoreQueueProvider.getTransportApiResponseProducer(); - TbQueueConsumer> consumer = tbCoreQueueProvider.getTransportApiRequestConsumer(); + TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); + TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); DefaultTbQueueResponseTemplate.DefaultTbQueueResponseTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueResponseTemplate.builder(); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e5de845fbc..3d47db432f 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,7 +25,6 @@ - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e731bd7bac..04c9205a12 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -550,26 +550,44 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - rule_engine: + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure_percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause_between_retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: # TODO 2.5: specify correct ENV variable names. + - + name: "Main" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - + name: "HighPriority" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 215bd85930..5ecb5fb415 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -145,11 +145,10 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + TbMsg tbMsg = TbMsg.newMsg( "CUSTOM", device.getId(), - new TbMsgMetaData(), TbMsgDataType.JSON, - "{}", null, null, null); + new TbMsgMetaData(), TbMsgDataType.JSON, "{}"); //TODO 2.5 // actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); @@ -261,12 +260,12 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + TbMsg tbMsg = TbMsg.newMsg( "CUSTOM", device.getId(), new TbMsgMetaData(), TbMsgDataType.JSON, - "{}", null, null, null); + "{}"); //TODO 2.5 // actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 43cdbe6ada..bc4db3b535 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -136,13 +136,12 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); // Pushing Message to the system - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), + TbMsg tbMsg = TbMsg.newMsg( "CUSTOM", device.getId(), new TbMsgMetaData(), TbMsgDataType.JSON, - "{}", - null, null, null); + "{}"); //TODO 2.5 // actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); diff --git a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java index 3c5f86a3fc..62881bd5af 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngineTest.java @@ -63,7 +63,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("70", actual.getMetaData().getValue("temp")); @@ -79,7 +79,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); assertEquals("94", actual.getMetaData().getValue("newAttr")); @@ -95,7 +95,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); TbMsg actual = scriptEngine.executeUpdate(msg); @@ -113,7 +113,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson); assertFalse(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -127,7 +127,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData,TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson); assertTrue(scriptEngine.executeFilter(msg)); scriptEngine.destroy(); } @@ -148,7 +148,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one"), actual); scriptEngine.destroy(); @@ -170,7 +170,7 @@ public class RuleNodeJsScriptEngineTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, null, null, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson); Set actual = scriptEngine.executeSwitch(msg); assertEquals(Sets.newHashSet("one", "three"), actual); scriptEngine.destroy(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7c7816900b..7d567a31f1 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -52,13 +52,34 @@ public final class TbMsg implements Serializable { //This field is not serialized because we use queues and there is no need to do it private final TbMsgCallback callback; - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); + } + + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + + public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { + return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), + data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + } + + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); } - public TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - TbMsgTransactionData transactionData, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, + TbMsgTransactionData transactionData, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this.id = id; this.type = type; this.originator = originator; @@ -115,7 +136,6 @@ public final class TbMsg implements Serializable { builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); return builder.build().toByteArray(); - } public static TbMsg fromBytes(byte[] data, TbMsgCallback callback) { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java index e9b1a890c9..0e8d66aa63 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java @@ -29,7 +29,7 @@ import java.util.Set; public final class PartitionChangeMsg implements TbActorMsg { @Getter - private final ServiceKey serviceKey; + private final ServiceQueueKey serviceQueueKey; @Getter private final Set partitions; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index 6f93301a48..49b30a5992 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import java.io.Serializable; +import java.util.Set; /** * Created by ashvayka on 15.03.18. @@ -31,9 +32,14 @@ public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; + private final Set relationTypes; @Override public MsgType getMsgType() { return MsgType.QUEUE_TO_RULE_ENGINE_MSG; } + + public boolean isTellNext() { + return relationTypes != null && !relationTypes.isEmpty(); + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java new file mode 100644 index 0000000000..be08f054b6 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import lombok.ToString; + +import java.util.Objects; + +@ToString +public class ServiceQueue { + + public static final String MAIN = "Main"; + + private final ServiceType type; + private final String queue; + + public ServiceQueue(ServiceType type) { + this.type = type; + this.queue = MAIN; + } + + public ServiceQueue(ServiceType type, String queue) { + this.type = type; + this.queue = queue; + } + + public ServiceType getType() { + return type; + } + + public String getQueue() { + return queue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ServiceQueue that = (ServiceQueue) o; + return type == that.type && + queue.equals(that.queue); + } + + @Override + public int hashCode() { + return Objects.hash(type, queue); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java similarity index 72% rename from common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java index aa6eb27323..f4bfefd878 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceKey.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java @@ -22,14 +22,15 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; @ToString -public class ServiceKey { +public class ServiceQueueKey { @Getter - private final ServiceType serviceType; + private final ServiceQueue serviceQueue; + @Getter private final TenantId tenantId; - public ServiceKey(ServiceType serviceType, TenantId tenantId) { - this.serviceType = serviceType; + public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) { + this.serviceQueue = serviceQueue; this.tenantId = tenantId; } @@ -37,13 +38,17 @@ public class ServiceKey { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ServiceKey that = (ServiceKey) o; - return serviceType == that.serviceType && + ServiceQueueKey that = (ServiceQueueKey) o; + return serviceQueue.equals(that.serviceQueue) && Objects.equals(tenantId, that.tenantId); } @Override public int hashCode() { - return Objects.hash(serviceType, tenantId); + return Objects.hash(serviceQueue, tenantId); + } + + public ServiceType getServiceType() { + return serviceQueue.getType(); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index a24a8c4b7c..0ca776126d 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; public static ServiceType of(String serviceType) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java similarity index 90% rename from common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java index 0d22db09e3..552eca84a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/MultipleTbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.common; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import java.util.concurrent.atomic.AtomicInteger; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java similarity index 88% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java index 29a17c9e99..c372f64b3d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.common; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import java.util.concurrent.atomic.AtomicInteger; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index e747b2f99d..f0039d0f38 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import java.util.Set; @@ -25,10 +25,10 @@ import java.util.Set; public class ClusterTopologyChangeEvent extends ApplicationEvent { @Getter - private final Set serviceKeys; + private final Set serviceQueueKeys; - public ClusterTopologyChangeEvent(Object source, Set serviceKeys) { + public ClusterTopologyChangeEvent(Object source, Set serviceQueueKeys) { super(source); - this.serviceKeys = serviceKeys; + this.serviceQueueKeys = serviceQueueKeys; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index fdf79d4e1b..1337a1cc74 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -24,7 +24,8 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -53,10 +54,6 @@ public class ConsistentHashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; - @Value("${queue.rule_engine.topic}") - private String ruleEngineTopic; - @Value("${queue.rule_engine.partitions:100}") - private Integer ruleEnginePartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @Value("${queue.partitions.virtual_nodes_size:16}") @@ -64,9 +61,9 @@ public class ConsistentHashPartitionService implements PartitionService { private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; - private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); - private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); @@ -85,49 +82,53 @@ public class ConsistentHashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); - partitionSizes.put(ServiceType.TB_CORE, corePartitions); - partitionSizes.put(ServiceType.TB_RULE_ENGINE, ruleEnginePartitions); - partitionTopics.put(ServiceType.TB_CORE, coreTopic); - partitionTopics.put(ServiceType.TB_RULE_ENGINE, ruleEngineTopic); + partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); + partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); } +// public Set getCurrentPartitions(ServiceType serviceType) { +// ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); +// TenantId tenantId = getSystemOrIsolatedTenantId(currentService); +// ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceType, tenantId); +// List partitions = myPartitions.get(serviceQueueKey); +// Set topicPartitions = new LinkedHashSet<>(); +// for (Integer partition : partitions) { +// TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); +// tpi.topic(partitionTopics.get(serviceType)); +// tpi.partition(partition); +// if (!tenantId.isNullUid()) { +// tpi.tenantId(tenantId); +// } +// topicPartitions.add(tpi.build()); +// } +// return topicPartitions; +// } + @Override - public Set getCurrentPartitions(ServiceType serviceType) { - ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); - TenantId tenantId = getSystemOrIsolatedTenantId(currentService); - ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); - List partitions = myPartitions.get(serviceKey); - Set topicPartitions = new LinkedHashSet<>(); - for (Integer partition : partitions) { - TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceType)); - tpi.partition(partition); - if (!tenantId.isNullUid()) { - tpi.tenantId(tenantId); - } - topicPartitions.add(tpi.build()); - } - return topicPartitions; + public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType), tenantId, entityId); } - //TODO 2.5 This should return cached TopicPartitionInfo objects instead of creating new one every time. @Override - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { + public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { + return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId); + } + + private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) { int hash = hashFunction.newHasher() .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - int partition = Math.abs(hash % partitionSizes.get(serviceType)); - boolean isolatedTenant = isIsolated(serviceType, tenantId); - TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceType, isolatedTenant ? tenantId : null, partition); - return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceType, tenantId, partition)); + int partition = Math.abs(hash % partitionSizes.get(serviceQueue)); + boolean isolatedTenant = isIsolated(serviceQueue, tenantId); + TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); + return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); } @Override public void recalculatePartitions(ServiceInfo currentService, List otherServices) { logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); - - Map> circles = new HashMap<>(); + Map> circles = new HashMap<>(); addNode(circles, currentService); for (ServiceInfo other : otherServices) { TenantId tenantId = getSystemOrIsolatedTenantId(other); @@ -140,26 +141,26 @@ public class ConsistentHashPartitionService implements PartitionService { } } - ConcurrentMap> oldPartitions = myPartitions; + ConcurrentMap> oldPartitions = myPartitions; TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); myPartitions = new ConcurrentHashMap<>(); partitionSizes.forEach((type, size) -> { - ServiceKey myServiceKey = new ServiceKey(type, myTenantId); + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myTenantId); for (int i = 0; i < size; i++) { - ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceKey), i); + ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); if (currentService.equals(serviceInfo)) { - ServiceKey serviceKey = new ServiceKey(type, getSystemOrIsolatedTenantId(serviceInfo)); - myPartitions.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(i); + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(type, getSystemOrIsolatedTenantId(serviceInfo)); + myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i); } } }); - myPartitions.forEach((serviceKey, partitions) -> { - if (!partitions.equals(oldPartitions.get(serviceKey))) { - log.info("[{}] NEW PARTITIONS: {}", serviceKey, partitions); + myPartitions.forEach((serviceQueueKey, partitions) -> { + if (!partitions.equals(oldPartitions.get(serviceQueueKey))) { + log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions); Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(serviceKey, partition)) + .map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)) .collect(Collectors.toSet()); - applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceKey, tpiList)); + applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList)); } }); tpiCache.clear(); @@ -167,14 +168,13 @@ public class ConsistentHashPartitionService implements PartitionService { if (currentOtherServices == null) { currentOtherServices = new ArrayList<>(otherServices); } else { - Set changes = new HashSet<>(); - Map> currentMap = getServiceKeyListMap(currentOtherServices); - Map> newMap = getServiceKeyListMap(otherServices); + Set changes = new HashSet<>(); + Map> currentMap = getServiceKeyListMap(currentOtherServices); + Map> newMap = getServiceKeyListMap(otherServices); currentOtherServices = otherServices; currentMap.forEach((key, list) -> { if (!list.equals(newMap.get(key))) { changes.add(key); - } }); currentMap.keySet().forEach(newMap::remove); @@ -214,13 +214,20 @@ public class ConsistentHashPartitionService implements PartitionService { } } - private Map> getServiceKeyListMap(List services) { - final Map> currentMap = new HashMap<>(); + private Map> getServiceKeyListMap(List services) { + final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { for (String serviceTypeStr : serviceInfo.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - ServiceKey serviceKey = new ServiceKey(serviceType, getSystemOrIsolatedTenantId(serviceInfo)); - currentMap.computeIfAbsent(serviceKey, key -> new ArrayList<>()).add(serviceInfo); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo)); + currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo); + } } }); return currentMap; @@ -230,20 +237,20 @@ public class ConsistentHashPartitionService implements PartitionService { return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceKey serviceKey, int partition) { - return buildTopicPartitionInfo(serviceKey.getServiceType(), serviceKey.getTenantId(), partition); + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) { + return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition); } - private TopicPartitionInfo buildTopicPartitionInfo(ServiceType serviceType, TenantId tenantId, int partition) { + private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) { TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); - tpi.topic(partitionTopics.get(serviceType)); + tpi.topic(partitionTopics.get(serviceQueue)); tpi.partition(partition); - ServiceKey myPartitionsSearchKey; - if (isIsolated(serviceType, tenantId)) { + ServiceQueueKey myPartitionsSearchKey; + if (isIsolated(serviceQueue, tenantId)) { tpi.tenantId(tenantId); - myPartitionsSearchKey = new ServiceKey(serviceType, tenantId); + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId); } else { - myPartitionsSearchKey = new ServiceKey(serviceType, new TenantId(TenantId.NULL_UUID)); + myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, new TenantId(TenantId.NULL_UUID)); } List partitions = myPartitions.get(myPartitionsSearchKey); if (partitions != null) { @@ -254,8 +261,8 @@ public class ConsistentHashPartitionService implements PartitionService { return tpi.build(); } - private boolean isIsolated(ServiceType serviceType, TenantId tenantId) { - return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceType); + private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { + return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceQueue.getType()); } private void logServiceInfo(TransportProtos.ServiceInfo server) { @@ -271,13 +278,24 @@ public class ConsistentHashPartitionService implements PartitionService { return new TenantId(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB())); } - private void addNode(Map> circles, ServiceInfo instance) { + private void addNode(Map> circles, ServiceInfo instance) { TenantId tenantId = getSystemOrIsolatedTenantId(instance); for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); - ServiceKey serviceKey = new ServiceKey(serviceType, tenantId); - for (int i = 0; i < virtualNodesSize; i++) { - circles.computeIfAbsent(serviceKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { + for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId); + partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions()); + partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic()); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } + } + } else { + ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId); + for (int i = 0; i < virtualNodesSize; i++) { + circles.computeIfAbsent(serviceQueueKey, key -> new ConsistentHashCircle<>()).put(hash(instance, i).asLong(), instance); + } } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 2ced989525..15cf33bb82 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -17,12 +17,16 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PostConstruct; import java.net.InetAddress; @@ -49,6 +53,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.tenant_id:}") private String tenantIdStr; + @Autowired(required = false) + private TbQueueRuleEngineSettings ruleEngineSettings; + private List serviceTypes; private ServiceInfo serviceInfo; @@ -78,13 +85,18 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { } builder.setTenantIdMSB(tenantId.getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getLeastSignificantBits()); - serviceInfo = builder.build(); - } + if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) { + for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) { + TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder() + .setName(queue.getName()) + .setTopic(queue.getTopic()) + .setPartitions(queue.getPartitions()).build(); + builder.addRuleEngineQueues(queueInfo); + } + } - @Override - public List getSupportedServiceTypes() { - return serviceTypes; + serviceInfo = builder.build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index 05218c68d4..19b8f665a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -17,7 +17,8 @@ package org.thingsboard.server.queue.discovery; import lombok.Getter; import org.springframework.context.ApplicationEvent; -import org.thingsboard.server.common.msg.queue.ServiceKey; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; @@ -26,13 +27,17 @@ import java.util.Set; public class PartitionChangeEvent extends ApplicationEvent { @Getter - private final ServiceKey serviceKey; + private final ServiceQueueKey serviceQueueKey; @Getter private final Set partitions; - public PartitionChangeEvent(Object source, ServiceKey serviceKey, Set partitions) { + public PartitionChangeEvent(Object source, ServiceQueueKey serviceQueueKey, Set partitions) { super(source); - this.serviceKey = serviceKey; + this.serviceQueueKey = serviceQueueKey; this.partitions = partitions; } + + public ServiceType getServiceType() { + return serviceQueueKey.getServiceQueue().getType(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b22b53f93e..e3b3e76b80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -29,10 +29,10 @@ import java.util.Set; */ public interface PartitionService { - Set getCurrentPartitions(ServiceType serviceType); - TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 8e20b35bde..053dd644c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -15,15 +15,10 @@ */ package org.thingsboard.server.queue.discovery; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; -import java.util.List; - public interface TbServiceInfoProvider { - List getSupportedServiceTypes(); - String getServiceId(); ServiceInfo getServiceInfo(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java index 2d51bc581a..37661e85e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java @@ -17,13 +17,13 @@ package org.thingsboard.server.queue.discovery; import lombok.AllArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.ServiceQueue; import java.util.Objects; @AllArgsConstructor public class TopicPartitionInfoKey { - private ServiceType serviceType; + private ServiceQueue serviceQueue; private TenantId isolatedTenantId; private int partition; @@ -33,12 +33,12 @@ public class TopicPartitionInfoKey { if (o == null || getClass() != o.getClass()) return false; TopicPartitionInfoKey that = (TopicPartitionInfoKey) o; return partition == that.partition && - serviceType == that.serviceType && + serviceQueue.equals(that.serviceQueue) && Objects.equals(isolatedTenantId, that.isolatedTenantId); } @Override public int hashCode() { - return Objects.hash(serviceType, isolatedTenantId, partition); + return Objects.hash(serviceQueue, isolatedTenantId, partition); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java similarity index 76% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 206a71d85e..72d5731139 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -21,14 +21,15 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -36,7 +37,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbQueueCoreSettings coreSettings; @@ -47,12 +48,12 @@ public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE private final TbAwsSqsSettings sqsSettings; private final TbQueueAdmin admin; - public AwsSqsMonolithQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings) { + public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,64 +65,64 @@ public class AwsSqsMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java similarity index 76% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 052d3fc0e3..1fba780395 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -26,10 +26,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -40,7 +40,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") -public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { +public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbAwsSqsSettings sqsSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; @@ -52,12 +52,12 @@ public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueAdmin admin; - public AwsSqsTbCoreQueueProvider(TbAwsSqsSettings sqsSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -68,50 +68,50 @@ public class AwsSqsTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java similarity index 74% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index d9f2cec9f4..4181a9442d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -24,12 +24,13 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -37,7 +38,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") -public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { +public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbQueueCoreSettings coreSettings; @@ -46,10 +47,10 @@ public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide private final TbAwsSqsSettings sqsSettings; private final TbQueueAdmin admin; - public AwsSqsTbRuleEngineQueueProvider(PartitionService partitionService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings) { + public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsSettings sqsSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -59,37 +60,37 @@ public class AwsSqsTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 01360ebed8..09ca194724 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -23,8 +23,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -36,17 +36,17 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { +public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; - public AwsSqsTransportQueueProvider(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider) { + public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbAwsSqsSettings sqsSettings, + TbServiceInfoProvider serviceInfoProvider) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; @@ -55,7 +55,7 @@ public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbAwsSqsProducerTemplate> producerTemplate = new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); @@ -76,17 +76,17 @@ public class AwsSqsTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java similarity index 70% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 5e1bab5e69..b40876d661 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -26,29 +26,30 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryMonolithQueueProvider(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + public InMemoryMonolithQueueFactory(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -56,57 +57,57 @@ public class InMemoryMonolithQueueProvider implements TbCoreQueueProvider, TbRul } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic() + ".notifications"); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(coreSettings.getTopic() + ".notifications"); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new InMemoryTbQueueConsumer<>(coreSettings.getTopic() + ".notifications"); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java similarity index 77% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 17e38b244e..9914cf65f9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -20,12 +20,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -39,17 +39,17 @@ import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvider { +public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings notificationSettings; - public InMemoryTbTransportQueueProvider(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + public InMemoryTbTransportQueueFactory(TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings notificationSettings) { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -57,7 +57,7 @@ public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvide } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); @@ -73,17 +73,17 @@ public class InMemoryTbTransportQueueProvider implements TbTransportQueueProvide } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new InMemoryTbQueueConsumer<>(notificationSettings.getNotificationsTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java similarity index 82% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 56cf4fb55f..8b84d08bff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,21 +26,22 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; @@ -50,12 +51,12 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaMonolithQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -66,7 +67,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); @@ -75,7 +76,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); @@ -84,7 +85,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); @@ -93,7 +94,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); @@ -102,7 +103,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); @@ -111,18 +112,19 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("monolith-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -133,7 +135,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); @@ -144,7 +146,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -155,7 +157,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); @@ -166,7 +168,7 @@ public class KafkaMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEn } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java similarity index 85% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 110a98b3b9..7083553abb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,10 +26,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") -public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { +public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; @@ -48,11 +48,11 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - public KafkaTbCoreQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings) { + public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -62,7 +62,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); @@ -71,7 +71,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); @@ -80,7 +80,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); @@ -89,7 +89,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); @@ -98,7 +98,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); @@ -107,7 +107,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); @@ -118,7 +118,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); @@ -129,7 +129,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); @@ -140,7 +140,7 @@ public class KafkaTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java similarity index 80% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 7f939ee11d..9b8a9c4ba0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -24,19 +24,20 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") -public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { +public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final PartitionService partitionService; private final TbKafkaSettings kafkaSettings; @@ -44,10 +45,10 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - public KafkaTbRuleEngineQueueProvider(PartitionService partitionService, TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings) { + public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -56,7 +57,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); @@ -65,7 +66,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); @@ -74,7 +75,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); @@ -84,7 +85,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); @@ -93,7 +94,7 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); @@ -102,18 +103,19 @@ public class KafkaTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + String queueName = configuration.getName(); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); - consumerBuilder.clientId("tb-rule-engine-consumer-" + serviceInfoProvider.getServiceId()); - consumerBuilder.groupId("tb-rule-engine-node-" + serviceInfoProvider.getServiceId()); + consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 2213889964..12f677af9d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -19,12 +19,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -41,7 +41,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { +public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbKafkaSettings kafkaSettings; private final TbServiceInfoProvider serviceInfoProvider; @@ -50,12 +50,12 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - public KafkaTbTransportQueueProvider(TbKafkaSettings kafkaSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -65,7 +65,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); @@ -90,7 +90,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-rule-engine-"+ serviceInfoProvider.getServiceId()); @@ -99,7 +99,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); @@ -108,7 +108,7 @@ public class KafkaTbTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 2dc9549679..4aa35e5d68 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,11 +27,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,10 +39,11 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleEngineQueueProvider { +public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -55,13 +56,13 @@ public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE private TbQueueProducer> tbCoreProducer; - public PubSubMonolithQueueProvider(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -73,65 +74,65 @@ public class PubSubMonolithQueueProvider implements TbCoreQueueProvider, TbRuleE } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java similarity index 78% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 4c89b35dec..da2fe3acdf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -27,9 +27,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") -public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { +public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -48,12 +48,12 @@ public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - public PubSubTbCoreQueueProvider(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueAdmin admin, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -63,51 +63,51 @@ public class PubSubTbCoreQueueProvider implements TbCoreQueueProvider { } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToCoreMsgConsumer() { + public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getTransportApiRequestConsumer() { + public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueProducer> getTransportApiResponseProducer() { + public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java similarity index 74% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 3f707235fa..da65d583c1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -25,19 +25,20 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") -public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvider { +public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -46,12 +47,12 @@ public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - public PubSubTbRuleEngineQueueProvider(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueAdmin admin, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueAdmin admin, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -61,39 +62,39 @@ public class PubSubTbRuleEngineQueueProvider implements TbRuleEngineQueueProvide } @Override - public TbQueueProducer> getTransportNotificationsMsgProducer() { + public TbQueueProducer> createTransportNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getToRuleEngineMsgConsumer() { + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override - public TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer() { + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java similarity index 79% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index b1a7efc950..f15faa11da 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -25,12 +25,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -42,7 +42,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j -public class PubSubTransportQueueProvider implements TbTransportQueueProvider { +public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbServiceInfoProvider serviceInfoProvider; @@ -52,12 +52,12 @@ public class PubSubTransportQueueProvider implements TbTransportQueueProvider { private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueAdmin admin; - public PubSubTransportQueueProvider(TbPubSubSettings pubSubSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -68,7 +68,7 @@ public class PubSubTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate() { + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), @@ -86,17 +86,17 @@ public class PubSubTransportQueueProvider implements TbTransportQueueProvider { } @Override - public TbQueueProducer> getRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> getTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } @Override - public TbQueueConsumer> getTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index e74c07152b..1a3ba1ab98 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -29,70 +29,70 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueProvider { +public interface TbCoreQueueFactory { /** * Used to push messages to instances of TB Transport Service * * @return */ - TbQueueProducer> getTransportNotificationsMsgProducer(); + TbQueueProducer> createTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineMsgProducer(); + TbQueueProducer> createRuleEngineMsgProducer(); /** * Used to push notifications to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); /** * Used to push messages to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> createTbCoreMsgProducer(); /** * Used to push notifications to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); + TbQueueProducer> createTbCoreNotificationsMsgProducer(); /** * Used to consume messages by TB Core Service * * @return */ - TbQueueConsumer> getToCoreMsgConsumer(); + TbQueueConsumer> createToCoreMsgConsumer(); /** * Used to consume high priority messages by TB Core Service * * @return */ - TbQueueConsumer> getToCoreNotificationsMsgConsumer(); + TbQueueConsumer> createToCoreNotificationsMsgConsumer(); /** * Used to consume Transport API Calls * * @return */ - TbQueueConsumer> getTransportApiRequestConsumer(); + TbQueueConsumer> createTransportApiRequestConsumer(); /** * Used to push replies to Transport API Calls * * @return */ - TbQueueProducer> getTransportApiResponseProducer(); + TbQueueProducer> createTransportApiResponseProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index c83159242d..9fe5de0bcc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -27,24 +27,24 @@ import javax.annotation.PostConstruct; @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { - private final TbCoreQueueProvider tbQueueProvider; + private final TbCoreQueueFactory tbQueueProvider; private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; - public TbCoreQueueProducerProvider(TbCoreQueueProvider tbQueueProvider) { + public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; } @PostConstruct public void init() { - this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index ed7dd43f2a..69de7ca3e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -27,7 +27,7 @@ import javax.annotation.PostConstruct; @ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { - private final TbRuleEngineQueueProvider tbQueueProvider; + private final TbRuleEngineQueueFactory tbQueueProvider; private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; @@ -35,17 +35,17 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; - public TbRuleEngineProducerProvider(TbRuleEngineQueueProvider tbQueueProvider) { + public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; } @PostConstruct public void init() { - this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); - this.toTransport = tbQueueProvider.getTransportNotificationsMsgProducer(); - this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); - this.toRuleEngineNotifications = tbQueueProvider.getRuleEngineNotificationsMsgProducer(); - this.toTbCoreNotifications = tbQueueProvider.getTbCoreNotificationsMsgProducer(); + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toTransport = tbQueueProvider.createTransportNotificationsMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); + this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java similarity index 75% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index fafa7da9e3..300fb986a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -23,60 +23,63 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueProvider { +public interface TbRuleEngineQueueFactory { /** * Used to push messages to instances of TB Transport Service * * @return */ - TbQueueProducer> getTransportNotificationsMsgProducer(); + TbQueueProducer> createTransportNotificationsMsgProducer(); /** * Used to push messages to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineMsgProducer(); + TbQueueProducer> createRuleEngineMsgProducer(); /** * Used to push notifications to instances of TB RuleEngine Service * * @return */ - TbQueueProducer> getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> createRuleEngineNotificationsMsgProducer(); /** * Used to push messages to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> createTbCoreMsgProducer(); /** * Used to push notifications to other instances of TB Core Service * * @return */ - TbQueueProducer> getTbCoreNotificationsMsgProducer(); + TbQueueProducer> createTbCoreNotificationsMsgProducer(); /** * Used to consume messages by TB Core Service * * @return + * @param configuration */ - TbQueueConsumer> getToRuleEngineMsgConsumer(); + //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary + TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration); /** * Used to consume high priority messages by TB Core Service * * @return */ - TbQueueConsumer> getToRuleEngineNotificationsMsgConsumer(); + TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java similarity index 78% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index 09521692f6..dc1d2c449c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -25,14 +25,14 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -public interface TbTransportQueueProvider { +public interface TbTransportQueueFactory { - TbQueueRequestTemplate, TbProtoQueueMsg> getTransportApiRequestTemplate(); + TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); - TbQueueProducer> getRuleEngineMsgProducer(); + TbQueueProducer> createRuleEngineMsgProducer(); - TbQueueProducer> getTbCoreMsgProducer(); + TbQueueProducer> createTbCoreMsgProducer(); - TbQueueConsumer> getTransportNotificationsConsumer(); + TbQueueConsumer> createTransportNotificationsConsumer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index d78f10ef74..0ebd59ec24 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -28,19 +28,19 @@ import javax.annotation.PostConstruct; @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { - private final TbTransportQueueProvider tbQueueProvider; + private final TbTransportQueueFactory tbQueueProvider; private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; - public TbTransportQueueProducerProvider(TbTransportQueueProvider tbQueueProvider) { + public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; } @PostConstruct public void init() { - this.toTbCore = tbQueueProvider.getTbCoreMsgProducer(); - this.toRuleEngine = tbQueueProvider.getRuleEngineMsgProducer(); + this.toTbCore = tbQueueProvider.createTbCoreMsgProducer(); + this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 4957b5ed5b..1c260c754d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java new file mode 100644 index 0000000000..48eeb5ba2a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Slf4j +@Data +@Configuration +@ConfigurationProperties(prefix = "queue.rule-engine") +public class TbQueueRuleEngineSettings { + + private String topic; + private List queues; + + //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. See how ther are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers + @PostConstruct + public void validate() { + queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { + log.warn("Main queue is not configured in thingsboard.yml"); + return new RuntimeException("No \"Main\" queue configured!"); + }); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java similarity index 96% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 263cf262a0..5ee58ab17e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index 0f2ac55862..50da4f4a1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java new file mode 100644 index 0000000000..c1a8fd883d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; + +@Data +public class TbRuleEngineQueueAckStrategyConfiguration { + +// @Value("${type}") + private String type; +// @Value("${retries:3}") + private int retries; +// @Value("${failure_percentage:0}") + private double failurePercentage; +// @Value("${pause_between_retries:3}") + private long pauseBetweenRetries; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java similarity index 71% rename from common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java rename to common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index fc3a36a88a..f89a615d7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -13,19 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue; +package org.thingsboard.server.queue.settings; import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; @Data -@Component -public class TbQueueRuleEngineSettings { +public class TbRuleEngineQueueConfiguration { - @Value("${queue.rule_engine.topic}") + private String name; private String topic; - - @Value("${queue.rule_engine.partitions}") + private int pollInterval; private int partitions; + private String packProcessingTimeout; + private TbRuleEngineQueueAckStrategyConfiguration ackStrategy; + } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 87c256174c..55d376b63b 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -19,6 +19,12 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +message QueueInfo { + string name = 1; + string topic = 2; + int32 partitions = 3; +} + /** * Service Discovery Data Structures; */ @@ -27,6 +33,7 @@ message ServiceInfo { repeated string serviceTypes = 2; int64 tenantIdMSB = 3; int64 tenantIdLSB = 4; + repeated QueueInfo ruleEngineQueues = 5; } /** @@ -380,6 +387,7 @@ message ToRuleEngineMsg { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; bytes tbMsg = 3; + repeated string relationTypes = 4; } message ToRuleEngineNotificationMsg { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3a707d11a7..d107bbbeeb 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -23,9 +23,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.queue.TbQueueCallback; @@ -46,7 +44,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.provider.TbTransportQueueProvider; +import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -92,7 +90,7 @@ public class DefaultTransportService implements TransportService { private int notificationsPollDuration; private final Gson gson = new Gson(); - private final TbTransportQueueProvider queueProvider; + private final TbTransportQueueFactory queueProvider; private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; @@ -112,7 +110,7 @@ public class DefaultTransportService implements TransportService { private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TbTransportQueueProvider queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + public DefaultTransportService(TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { this.queueProvider = queueProvider; this.producerProvider = producerProvider; this.partitionService = partitionService; @@ -128,10 +126,10 @@ public class DefaultTransportService implements TransportService { this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("transport-scheduler")); this.transportCallbackExecutor = Executors.newWorkStealingPool(20); this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); - transportApiRequestTemplate = queueProvider.getTransportApiRequestTemplate(); + transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); - transportNotificationsConsumer = queueProvider.getTransportNotificationsConsumer(); + transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); transportNotificationsConsumer.subscribe(); transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { @@ -251,8 +249,7 @@ public class DefaultTransportService implements TransportService { metaData.putValue("deviceType", sessionInfo.getDeviceType()); metaData.putValue("ts", tsKv.getTs() + ""); JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList()); - TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), - deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, metaData, gson.toJson(json)); sendToRuleEngine(tenantId, tbMsg, packCallback); } } @@ -268,8 +265,7 @@ public class DefaultTransportService implements TransportService { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("deviceName", sessionInfo.getDeviceName()); metaData.putValue("deviceType", sessionInfo.getDeviceType()); - TbMsg tbMsg = new TbMsg(UUID.randomUUID(), SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, - TbMsgDataType.JSON, gson.toJson(json), null, null, TbMsgCallback.EMPTY); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), deviceId, metaData, gson.toJson(json)); sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 38914f4553..f46878ca68 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -45,27 +46,80 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import java.util.Set; +import java.util.function.Consumer; /** * Created by ashvayka on 13.01.18. */ public interface TbContext { + /* + * + * METHODS TO CONTROL THE MESSAGE FLOW + * + */ + + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using specified relationType. + * + * @param msg + * @param relationType + */ void tellNext(TbMsg msg, String relationType); - void tellNext(TbMsg msg, String relationType, Throwable th); - + /** + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using one of specified relationTypes. + * + * @param msg + * @param relationTypes + */ void tellNext(TbMsg msg, Set relationTypes); + /** + * Sends message to the current Rule Node with specified delay in milliseconds. + * Note: this message is not queued and may be lost in case of a server restart. + * + * @param msg + */ void tellSelf(TbMsg msg, long delayMs); - boolean isLocalEntity(EntityId entityId); - + /** + * Notifies Rule Engine about failure to process current message. + * + * @param msg - message + * @param th - exception + */ void tellFailure(TbMsg msg, Throwable th); - void updateSelf(RuleNode self); + /** + * Puts new message to queue for processing by the Root Rule Chain + * + * @param msg - message + */ + void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); + + /** + * Puts new message to custom queue for processing + * + * @param msg - message + */ + void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String relationType); + + void enqueueForTellNext(TbMsg msg, Set relationTypes); + + void enqueueForTellNext(TbMsg msg, String relationType, Runnable onSuccess, Consumer onFailure); - void sendTbMsgToRuleEngine(TbMsg msg); + void enqueueForTellNext(TbMsg msg, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure); + + void enqueueForTellNext(TbMsg msg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure); + + void ack(TbMsg tbMsg); TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data); @@ -77,8 +131,17 @@ public interface TbContext { TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId); + // TODO: Does this changes the message? TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId); + /* + * + * METHODS TO PROCESS THE MESSAGES + * + */ + + boolean isLocalEntity(EntityId entityId); + RuleNodeId getSelfId(); TenantId getTenantId(); @@ -129,7 +192,7 @@ public interface TbContext { void logJsEvalFailure(); - String getNodeId(); + String getServiceId(); RuleChainTransactionService getRuleChainTransactionService(); @@ -139,7 +202,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); + //TODO 2.5: - need to remove this. RedisTemplate getRedisTemplate(); - String getServerAddress(); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index 6320d5625f..afc4181f70 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import javax.script.ScriptException; - import static org.thingsboard.common.util.DonAsynchron.withCallback; @@ -63,16 +61,16 @@ public abstract class TbAbstractAlarmNode ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Created"), + throwable -> ctx.tellFailure(toAlarmMsg(ctx, alarmResult, msg), throwable)); } else if (alarmResult.isUpdated) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Updated"); } else if (alarmResult.isCleared) { ctx.tellNext(toAlarmMsg(ctx, alarmResult, msg), "Cleared"); } }, - t -> ctx.tellFailure(msg, t) - , ctx.getDbCallbackExecutor()); + t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } protected abstract ListenableFuture processAlarm(TbContext ctx, TbMsg msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index 3ee2127fde..7b84fa06a9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -122,7 +122,9 @@ public abstract class TbAbstractCustomerActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); return Optional.of(savedCustomer.getId()); } return Optional.empty(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java index bcfabf1100..96fdd2a11e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java @@ -186,7 +186,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Device Created message: {}", savedDevice), + throwable -> log.warn("Failed to push Device Created message: {}", savedDevice, throwable)); targetEntity.setEntityId(savedDevice.getId()); } break; @@ -201,7 +203,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Asset Created message: {}", savedAsset), + throwable -> log.warn("Failed to push Asset Created message: {}", savedAsset, throwable)); targetEntity.setEntityId(savedAsset.getId()); } break; @@ -215,7 +219,9 @@ public abstract class TbAbstractRelationActionNode log.trace("Pushed Customer Created message: {}", savedCustomer), + throwable -> log.warn("Failed to push Customer Created message: {}", savedCustomer, throwable)); targetEntity.setEntityId(savedCustomer.getId()); } break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 1952985e53..282ca00639 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -68,7 +68,7 @@ public class TbMsgCountNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) { JsonObject telemetryJson = new JsonObject(); - telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue()); + telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getServiceId(), messagesProcessed.longValue()); messagesProcessed = new AtomicLong(0); @@ -76,11 +76,12 @@ public class TbMsgCountNode implements TbNode { metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); //TODO 2.5: Callback? - TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, null); - ctx.tellNext(tbMsg, SUCCESS); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); + ctx.enqueueForTellNext(tbMsg, SUCCESS, null, null); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 0ef9c4d46a..5da20f5c35 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -73,7 +73,7 @@ public class TbMsgDelayNode implements TbNode { TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); } else { - ctx.tellNext(msg, FAILURE, new RuntimeException("Max limit of pending messages reached!")); + ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index fe765abdf9..e5b3f4a1c7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -66,7 +66,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getNodeId()) + .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -108,7 +108,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 216fea54e3..7c3705fa1f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.util.Properties; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; @Slf4j @RuleNode( @@ -55,7 +54,7 @@ public class TbKafkaNode implements TbNode { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbKafkaNodeConfiguration.class); Properties properties = new Properties(); - properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getNodeId()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, "producer-tb-kafka-node-" + ctx.getSelfId().getId().toString() + "-" + ctx.getServiceId()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, config.getValueSerializer()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, config.getKeySerializer()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java index 3349706221..015cced6c6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java @@ -105,7 +105,7 @@ class TbRedisQueueProcessor { } private String constructRedisKey() { - return ctx.getServerAddress() + ctx.getSelfId(); + return ctx.getServiceId() + ctx.getSelfId(); } private int validateMaxQueueSize() { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index a6bd7b6164..b8eff0901a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -58,8 +58,7 @@ public class TbSynchronizationBeginNode implements TbNode { TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); //TODO 2.5: Callback? - TbMsg tbMsg = new TbMsg(msg.getId(), msg.getType(), msg.getOriginator(), msg.getMetaData(), TbMsgDataType.JSON, - msg.getData(), transactionData, msg.getRuleChainId(), msg.getRuleNodeId(), null); + TbMsg tbMsg = TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index e0b76f1cda..c2177a9c9a 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -101,7 +101,7 @@ public class TbAlarmNodeTest { public void newAlarmCanBeCreated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -143,7 +143,7 @@ public class TbAlarmNodeTest { public void buildDetailsThrowsException() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFailedFuture(new NotImplementedException("message"))); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); @@ -166,7 +166,7 @@ public class TbAlarmNodeTest { public void ifAlarmClearedCreateNew() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); Alarm clearedAlarm = Alarm.builder().status(CLEARED_ACK).build(); @@ -211,7 +211,7 @@ public class TbAlarmNodeTest { public void alarmCanBeUpdated() throws ScriptException, IOException { initWithCreateAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); @@ -258,7 +258,7 @@ public class TbAlarmNodeTest { public void alarmCanBeCleared() throws ScriptException, IOException { initWithClearAlarmScript(); metaData.putValue("key", "value"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); long oldEndDate = System.currentTimeMillis(); Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index c840e708eb..ac91e0548c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -59,7 +59,7 @@ public class TbJsFilterNodeTest { @Test public void falseEvaluationDoNotSendMsg() throws TbNodeException, ScriptException { initWithScript(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", null, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(false)); @@ -72,7 +72,7 @@ public class TbJsFilterNodeTest { public void exceptionInJsThrowsException() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFailedFuture(new ScriptException("error"))); @@ -85,7 +85,7 @@ public class TbJsFilterNodeTest { public void metadataConditionCanBeTrue() throws TbNodeException, ScriptException { initWithScript(); TbMsgMetaData metaData = new TbMsgMetaData(); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeFilterAsync(msg)).thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index 4934e14062..f96797b69e 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -66,7 +66,7 @@ public class TbJsSwitchNodeTest { metaData.putValue("humidity", "99"); String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeSwitch(msg)).thenReturn(Sets.newHashSet("one", "three")); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index 8315d8e40b..c07bee7d79 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -63,7 +63,7 @@ public class TbMsgToEmailNodeTest { metaData.putValue("name", "temp"); metaData.putValue("passed", "5"); metaData.putValue("count", "100"); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", originator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); emailNode.onMsg(ctx, msg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 0169cadcb2..1575823988 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -103,7 +103,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -128,7 +128,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -153,7 +153,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(null)); @@ -167,7 +167,7 @@ public class TbGetCustomerAttributeNodeTest { @Test public void customerAttributeAddedInMetadata() { CustomerId customerId = new CustomerId(UUIDs.timeBased()); - msg = new TbMsg(UUIDs.timeBased(), "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "CUSTOMER", customerId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); entityAttributeFetched(customerId); } @@ -178,7 +178,7 @@ public class TbGetCustomerAttributeNodeTest { User user = new User(); user.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", userId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getUserService()).thenReturn(userService); when(userService.findUserByIdAsync(any(), eq(userId))).thenReturn(Futures.immediateFuture(user)); @@ -193,7 +193,7 @@ public class TbGetCustomerAttributeNodeTest { Asset asset = new Asset(); asset.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -208,7 +208,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); @@ -235,7 +235,7 @@ public class TbGetCustomerAttributeNodeTest { Device device = new Device(); device.setCustomerId(customerId); - msg = new TbMsg(UUIDs.timeBased(), "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + msg = TbMsg.newMsg( "USER", deviceId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getDeviceService()).thenReturn(deviceService); when(deviceService.findDeviceByIdAsync(any(), eq(deviceId))).thenReturn(Futures.immediateFuture(device)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 1ed66fc786..fa845c4a18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -92,7 +92,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(),eq( assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -120,7 +120,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(asset)); @@ -147,7 +147,7 @@ public class TbChangeOriginatorNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "ASSET", assetId, new TbMsgMetaData(), TbMsgDataType.JSON,"{}", ruleChainId, ruleNodeId); when(ctx.getAssetService()).thenReturn(assetService); when(assetService.findAssetByIdAsync(any(), eq(assetId))).thenReturn(Futures.immediateFuture(null)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index 9a315f893b..314ee78c9c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -63,8 +63,8 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId, null); - TbMsg transformedMsg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON,rawJson, ruleChainId, ruleNodeId); + TbMsg transformedMsg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, "{new}", ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFuture(transformedMsg)); @@ -85,7 +85,7 @@ public class TbTransformMsgNodeTest { RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); - TbMsg msg = new TbMsg(UUIDs.timeBased(), "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId, null); + TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); mockJsExecutor(); when(scriptEngine.executeUpdateAsync(msg)).thenReturn(Futures.immediateFailedFuture(new IllegalStateException("error"))); From c52d0d26d305a6d90f4e78c76fdb7dde5cfa1a35 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Apr 2020 19:23:33 +0300 Subject: [PATCH 041/196] Refactoring of Message Routing --- .../actors/ruleChain/DefaultTbContext.java | 5 ++++ .../rule/engine/api/TbContext.java | 9 ++++++ .../engine/action/TbAbstractAlarmNode.java | 2 ++ .../action/TbAbstractCustomerActionNode.java | 2 +- .../TbCopyAttributesToEntityViewNode.java | 8 +++--- .../rule/engine/action/TbLogNode.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 2 +- .../TbSaveToCustomCassandraTableNode.java | 6 ++-- .../rule/engine/aws/sns/TbSnsNode.java | 7 ++--- .../rule/engine/aws/sqs/TbSqsNode.java | 9 ++---- .../rule/engine/debug/TbMsgGeneratorNode.java | 2 +- .../rule/engine/delay/TbMsgDelayNode.java | 5 ++-- .../engine/filter/TbCheckAlarmStatusNode.java | 3 +- .../engine/filter/TbMsgTypeFilterNode.java | 2 +- .../engine/filter/TbMsgTypeSwitchNode.java | 2 +- .../filter/TbOriginatorTypeFilterNode.java | 2 +- .../rule/engine/gcp/pubsub/TbPubSubNode.java | 2 +- .../engine/geo/TbGpsGeofencingActionNode.java | 26 +++++++++++------ .../rule/engine/kafka/TbKafkaNode.java | 5 ++-- .../rule/engine/mail/TbSendEmailNode.java | 3 +- .../metadata/TbAbstractGetAttributesNode.java | 3 +- .../TbAbstractGetEntityDetailsNode.java | 2 +- .../engine/metadata/TbEntityGetAttrNode.java | 5 ++-- .../metadata/TbGetOriginatorFieldsNode.java | 4 +-- .../engine/metadata/TbGetTelemetryNode.java | 3 +- .../rule/engine/mqtt/TbMqttNode.java | 4 +-- .../rule/engine/rabbitmq/TbRabbitMqNode.java | 4 +-- .../rule/engine/rest/TbHttpClient.java | 2 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 1 + .../rule/engine/rpc/TbSendRPCRequestNode.java | 8 ++++-- .../engine/telemetry/TbMsgAttributesNode.java | 1 - .../engine/telemetry/TbMsgTimeseriesNode.java | 4 +-- .../telemetry/TelemetryNodeCallback.java | 4 +-- .../transform/TbAbstractTransformNode.java | 2 +- .../rule/engine/action/TbAlarmNodeTest.java | 28 ++++++++++++++----- .../engine/filter/TbJsFilterNodeTest.java | 11 ++++++-- .../engine/filter/TbJsSwitchNodeTest.java | 9 ++++-- .../engine/mail/TbMsgToEmailNodeTest.java | 1 - .../TbGetCustomerAttributeNodeTest.java | 13 +++++++-- .../transform/TbTransformMsgNodeTest.java | 10 +++++-- 40 files changed, 137 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 2f31d1dc35..f8ac564e42 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -90,6 +90,11 @@ class DefaultTbContext implements TbContext { this.nodeCtx = nodeCtx; } + @Override + public void tellSuccess(TbMsg msg) { + tellNext(msg, Collections.singleton(TbRelationTypes.SUCCESS), null); + } + @Override public void tellNext(TbMsg msg, String relationType) { tellNext(msg, Collections.singleton(relationType), null); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index f46878ca68..72a552cb4f 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -59,6 +59,15 @@ public interface TbContext { * */ + /** + * Indicates that message was successfully processed by the rule node. + * Sends message to all Rule Nodes in the Rule Chain + * that are connected to the current Rule Node using "Success" relationType. + * + * @param msg + */ + void tellSuccess(TbMsg msg); + /** * Sends message to all Rule Nodes in the Rule Chain * that are connected to the current Rule Node using specified relationType. diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index afc4181f70..b3344105bb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -68,6 +68,8 @@ public abstract class TbAbstractAlarmNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java index 7b84fa06a9..bdf8c5fe1e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractCustomerActionNode.java @@ -63,7 +63,7 @@ public abstract class TbAbstractCustomerActionNode ctx.tellNext(msg, "Success"), + m -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index f009b737e9..c5dc5cc5f4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -89,7 +89,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) { if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) || DataConstants.ACTIVITY_EVENT.equals(msg.getType()) || - SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType()) ) { + SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData())); List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList()); @@ -117,13 +117,14 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } List filteredAttributes = attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList()); - if (filteredAttributes != null && !filteredAttributes.isEmpty()) { + if (!filteredAttributes.isEmpty()) { ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes); transformAndTellNext(ctx, msg, entityView); } } } } + ctx.ack(msg); }, t -> ctx.tellFailure(msg, t)); } else { @@ -135,8 +136,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) { - TbMsg updMsg = ctx.transformMsg(msg, msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()); - ctx.tellNext(updMsg, SUCCESS); + ctx.enqueueForTellNext(ctx.newMsg(msg.getType(), entityView.getId(), msg.getMetaData(), msg.getData()), SUCCESS); } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java index 62410e3052..2831e890d6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbLogNode.java @@ -58,7 +58,7 @@ public class TbLogNode implements TbNode { toString -> { ctx.logJsEvalResponse(); log.info(toString); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); }, t -> { ctx.logJsEvalResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 282ca00639..10e664bc16 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -77,7 +77,7 @@ public class TbMsgCountNode implements TbNode { //TODO 2.5: Callback? TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); - ctx.enqueueForTellNext(tbMsg, SUCCESS, null, null); + ctx.enqueueForTellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); } else { messagesProcessed.incrementAndGet(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java index 50471b366a..aa7bfa81d1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbSaveToCustomCassandraTableNode.java @@ -105,10 +105,8 @@ public class TbSaveToCustomCassandraTableNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - withCallback(save(msg, ctx), aVoid -> { - ctx.tellNext(msg, SUCCESS); - }, e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); + public void onMsg(TbContext ctx, TbMsg msg) { + withCallback(save(msg, ctx), aVoid -> ctx.tellSuccess(msg), e -> ctx.tellFailure(msg, e), ctx.getDbCallbackExecutor()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java index 59d27018d7..d43ebb26ef 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sns/TbSnsNode.java @@ -74,11 +74,8 @@ public class TbSnsNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java index 37f4e4baee..3cc6074165 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/aws/sqs/TbSqsNode.java @@ -80,13 +80,10 @@ public class TbSqsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), - t -> { - TbMsg next = processException(ctx, msg, t); - ctx.tellFailure(next, t); - }); + ctx::tellSuccess, + t -> ctx.tellFailure(processException(ctx, msg, t), t)); } private ListenableFuture publishMessageAsync(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 93114c28e0..4f469678a2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -97,7 +97,7 @@ public class TbMsgGeneratorNode implements TbNode { withCallback(generate(ctx), m -> { if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { - ctx.tellNext(m, SUCCESS); + ctx.enqueueForTellNext(m, SUCCESS); scheduleTickMsg(ctx); currentMsgCount++; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 5da20f5c35..1807c5c4fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -65,13 +65,14 @@ public class TbMsgDelayNode implements TbNode { if (msg.getType().equals(TB_MSG_DELAY_NODE_MSG)) { TbMsg pendingMsg = pendingMsgs.remove(UUID.fromString(msg.getData())); if (pendingMsg != null) { - ctx.tellNext(pendingMsg, SUCCESS); + ctx.enqueueForTellNext(pendingMsg, SUCCESS); } } else { - if(pendingMsgs.size() < config.getMaxPendingMsgs()) { + if (pendingMsgs.size() < config.getMaxPendingMsgs()) { pendingMsgs.put(msg.getId(), msg); TbMsg tickMsg = ctx.newMsg(TB_MSG_DELAY_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), msg.getId().toString()); ctx.tellSelf(tickMsg, getDelay(msg)); + ctx.ack(msg); } else { ctx.tellFailure(msg, new RuntimeException("Max limit of pending messages reached!")); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index e0ec9300d4..8935cfc1f5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -76,14 +76,13 @@ public class TbCheckAlarmStatusNode implements TbNode { break; } } - if (isPresent) { ctx.tellNext(msg, "True"); } else { ctx.tellNext(msg, "False"); } } else { - ctx.tellFailure(msg, new TbNodeException("No such Alarm found.")); + ctx.tellFailure(msg, new TbNodeException("No such alarm found.")); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java index dd55ef2584..3799e60f73 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeFilterNode.java @@ -44,7 +44,7 @@ public class TbMsgTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { ctx.tellNext(msg, config.getMessageTypes().contains(msg.getType()) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index d0fe593711..07a0508a87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -45,7 +45,7 @@ public class TbMsgTypeSwitchNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String relationType; if (msg.getType().equals(SessionMsgType.POST_ATTRIBUTES_REQUEST.name())) { relationType = "Post attributes"; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java index ac3db9968f..de3a48142e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbOriginatorTypeFilterNode.java @@ -42,7 +42,7 @@ public class TbOriginatorTypeFilterNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { EntityType originatorType = msg.getOriginator().getEntityType(); ctx.tellNext(msg, config.getOriginatorTypes().contains(originatorType) ? "True" : "False"); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 8e3bcdb7dc..5a8b92cf80 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -99,7 +99,7 @@ public class TbPubSubNode implements TbNode { ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback() { public void onSuccess(String messageId) { TbMsg next = processPublishResult(ctx, msg, messageId); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java index e5b3f4a1c7..832a86b5cb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/geo/TbGpsGeofencingActionNode.java @@ -47,11 +47,12 @@ import java.util.concurrent.TimeoutException; type = ComponentType.ACTION, name = "gps geofencing events", configClazz = TbGpsGeofencingActionNodeConfiguration.class, - relationTypes = {"Entered", "Left", "Inside", "Outside"}, + relationTypes = {"Success", "Entered", "Left", "Inside", "Outside"}, nodeDescription = "Produces incoming messages using GPS based geofencing", nodeDetails = "Extracts latitude and longitude parameters from incoming message and returns different events based on configuration parameters", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeGpsGeofencingConfig") + configDirective = "tbActionNodeGpsGeofencingConfig" +) public class TbGpsGeofencingActionNode extends AbstractGeofencingNode { private final Map entityStates = new HashMap<>(); @@ -78,17 +79,26 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode (entityState.isInside() ? - TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { - setStaid(ctx, msg.getOriginator(), entityState); - ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } else { + if (!entityState.isStayed()) { + long stayTime = ts - entityState.getStateSwitchTime(); + if (stayTime > (entityState.isInside() ? + TimeUnit.valueOf(config.getMinInsideDurationTimeUnit()).toMillis(config.getMinInsideDuration()) : TimeUnit.valueOf(config.getMinOutsideDurationTimeUnit()).toMillis(config.getMinOutsideDuration()))) { + setStaid(ctx, msg.getOriginator(), entityState); + ctx.tellNext(msg, entityState.isInside() ? "Inside" : "Outside"); + told = true; + } } } + if (!told) { + ctx.tellSuccess(msg); + } } private void switchState(TbContext ctx, EntityId entityId, EntityGeofencingState entityState, boolean matches, long ts) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java index 7c3705fa1f..2c487f67b9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/kafka/TbKafkaNode.java @@ -64,8 +64,7 @@ public class TbKafkaNode implements TbNode { properties.put(ProducerConfig.LINGER_MS_CONFIG, config.getLinger()); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, config.getBufferMemory()); if (config.getOtherProperties() != null) { - config.getOtherProperties() - .forEach((k,v) -> properties.put(k, v)); + config.getOtherProperties().forEach(properties::put); } try { this.producer = new KafkaProducer<>(properties); @@ -75,7 +74,7 @@ public class TbKafkaNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(config.getTopicPattern(), msg.getMetaData()); try { producer.send(new ProducerRecord<>(topic, msg.getData()), diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java index 1c41c2a124..ee46b81116 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mail/TbSendEmailNode.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.Properties; import static org.thingsboard.common.util.DonAsynchron.withCallback; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @Slf4j @RuleNode( @@ -79,7 +78,7 @@ public class TbSendEmailNode implements TbNode { sendEmail(ctx, email); return null; }), - ok -> ctx.tellNext(msg, SUCCESS), + ok -> ctx.tellSuccess(msg), fail -> ctx.tellFailure(msg, fail)); } catch (Exception ex) { ctx.tellFailure(msg, ex); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index be4e6b4e30..8aadbd6b32 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -42,7 +42,6 @@ import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @@ -100,7 +99,7 @@ public abstract class TbAbstractGetAttributesNode ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java index b6fdaa565a..4ca0bf2dff 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetEntityDetailsNode.java @@ -57,7 +57,7 @@ public abstract class TbAbstractGetEntityDetailsNode ctx.tellNext(m, SUCCESS), + ctx::tellSuccess, t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java index ca2c99fe7a..8f19cba741 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbEntityGetAttrNode.java @@ -50,8 +50,7 @@ public abstract class TbEntityGetAttrNode implements TbNode @Override public void onMsg(TbContext ctx, TbMsg msg) { try { - withCallback( - findEntityAsync(ctx, msg.getOriginator()), + withCallback(findEntityAsync(ctx, msg.getOriginator()), entityId -> safeGetAttributes(ctx, msg, entityId), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { @@ -88,7 +87,7 @@ public abstract class TbEntityGetAttrNode implements TbNode String attrName = config.getAttrMapping().get(r.getKey()); msg.getMetaData().putValue(attrName, r.getValueAsString()); }); - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java index 759007789a..160ea9adb1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetOriginatorFieldsNode.java @@ -49,10 +49,10 @@ public class TbGetOriginatorFieldsNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { try { withCallback(putEntityFields(ctx, msg.getOriginator(), msg), - i -> ctx.tellNext(msg, SUCCESS), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); + i -> ctx.tellSuccess(msg), t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); } catch (Throwable th) { ctx.tellFailure(msg, th); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index caef71a4db..a3d84b25db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -106,8 +106,7 @@ public class TbGetTelemetryNode implements TbNode { ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); DonAsynchron.withCallback(list, data -> { process(data, msg); - TbMsg newMsg = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); - ctx.tellNext(newMsg, SUCCESS); + ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { ctx.tellFailure(msg, e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 755cd27a33..b13f72b238 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -72,12 +72,12 @@ public class TbMqttNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { String topic = TbNodeUtils.processPattern(this.config.getTopicPattern(), msg.getMetaData()); this.mqttClient.publish(topic, Unpooled.wrappedBuffer(msg.getData().getBytes(UTF8)), MqttQoS.AT_LEAST_ONCE) .addListener(future -> { if (future.isSuccess()) { - ctx.tellNext(msg, TbRelationTypes.SUCCESS); + ctx.tellSuccess(msg); } else { TbMsg next = processException(ctx, msg, future.cause()); ctx.tellFailure(next, future.cause()); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java index 3c3bde2915..4a61cf63f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rabbitmq/TbRabbitMqNode.java @@ -74,9 +74,9 @@ public class TbRabbitMqNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { withCallback(publishMessageAsync(ctx, msg), - m -> ctx.tellNext(m, TbRelationTypes.SUCCESS), + ctx::tellSuccess, t -> { TbMsg next = processException(ctx, msg, t); ctx.tellFailure(next, t); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 65eea0b4c9..5b30f4792c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -113,7 +113,7 @@ class TbHttpClient { queueProcessor.resetCounter(); } TbMsg next = processResponse(ctx, msg, responseEntity); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + ctx.tellSuccess(next); } else { if (config.isUseRedisQueueForMsgPersistence()) { processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 0c5c7d188e..393ef0bd7c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -59,6 +59,7 @@ public class TbSendRPCReplyNode implements TbNode { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { ctx.getRpcService().sendRpcReplyToDevice(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.tellSuccess(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index e4eeef2e5f..88b060fe5c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -112,13 +112,15 @@ public class TbSendRPCRequestNode implements TbNode { ctx.getRpcService().sendRpcRequestToDevice(request, ruleEngineDeviceRpcResponse -> { if (!ruleEngineDeviceRpcResponse.getError().isPresent()) { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); - ctx.tellNext(next, TbRelationTypes.SUCCESS); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), ruleEngineDeviceRpcResponse.getResponse().orElse("{}")); + ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); } else { - TbMsg next = ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); + TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); + ctx.enqueueForTellNext(next, TbRelationTypes.FAILURE); } }); + ctx.ack(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 4ddd082f3e..8d80e17f87 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -61,7 +61,6 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); return; } - String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index ce7edf1c5f..14565ac475 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -74,7 +74,7 @@ public class TbMsgTimeseriesNode implements TbNode { } String src = msg.getData(); Map> tsKvMap = JsonConverter.convertToTelemetry(new JsonParser().parse(src), ts); - if (tsKvMap == null) { + if (tsKvMap.isEmpty()) { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } @@ -85,7 +85,7 @@ public class TbMsgTimeseriesNode implements TbNode { } } String ttlValue = msg.getMetaData().getValue("TTL"); - long ttl = !StringUtils.isEmpty(ttlValue) ? Long.valueOf(ttlValue) : config.getDefaultTTL(); + long ttl = !StringUtils.isEmpty(ttlValue) ? Long.parseLong(ttlValue) : config.getDefaultTTL(); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), tsKvEntryList, ttl, new TelemetryNodeCallback(ctx, msg)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java index d545d41d7b..bd15c5bd83 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TelemetryNodeCallback.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.msg.TbMsg; import javax.annotation.Nullable; -import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; - /** * Created by ashvayka on 02.04.18. */ @@ -34,7 +32,7 @@ class TelemetryNodeCallback implements FutureCallback { @Override public void onSuccess(@Nullable Void result) { - ctx.tellNext(msg, SUCCESS); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java index e768f699cd..84a6b76920 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbAbstractTransformNode.java @@ -55,7 +55,7 @@ public abstract class TbAbstractTransformNode implements TbNode { protected void transformSuccess(TbContext ctx, TbMsg msg, TbMsg m) { if (m != null) { - ctx.tellNext(m, SUCCESS); + ctx.tellSuccess(m); } else { ctx.tellNext(msg, FAILURE); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index c2177a9c9a..2fcdc6c838 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -26,18 +26,19 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -47,13 +48,26 @@ import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; -import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.*; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_CLEARED_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_EXISTING_ALARM; +import static org.thingsboard.rule.engine.action.TbAbstractAlarmNode.IS_NEW_ALARM; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.CRITICAL; import static org.thingsboard.server.common.data.alarm.AlarmSeverity.WARNING; -import static org.thingsboard.server.common.data.alarm.AlarmStatus.*; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.ACTIVE_UNACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_ACK; +import static org.thingsboard.server.common.data.alarm.AlarmStatus.CLEARED_UNACK; @RunWith(MockitoJUnitRunner.class) public class TbAlarmNodeTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index ac91e0548c..b0f64fb980 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -19,7 +19,6 @@ import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -28,7 +27,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; @@ -39,7 +41,10 @@ import javax.script.ScriptException; import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsFilterNodeTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index f96797b69e..9286a06da6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -28,7 +28,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; @@ -41,7 +44,9 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class TbJsSwitchNodeTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java index c07bee7d79..70f58c5989 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mail/TbMsgToEmailNodeTest.java @@ -37,7 +37,6 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 1575823988..211459b84c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -31,8 +31,17 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index 314ee78c9c..3d40f3acd0 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -27,7 +27,10 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.TbMsg; @@ -39,7 +42,10 @@ import java.util.concurrent.Callable; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.same; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @RunWith(MockitoJUnitRunner.class) From ff3fd89acedf0b356db9638adbf38c3c8e41abfe Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Apr 2020 19:40:38 +0300 Subject: [PATCH 042/196] Added ability to process tellNext messages from Queue --- .../actors/ruleChain/DefaultTbContext.java | 6 +- .../RuleChainActorMessageProcessor.java | 66 ++++++++++--------- .../thingsboard/server/common/msg/TbMsg.java | 6 +- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index f8ac564e42..7178b94dfa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; @@ -175,6 +176,9 @@ class DefaultTbContext implements TbContext { } private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); + RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); + tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 00d9d85b1c..3ab67bf81f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,18 +34,18 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; import java.util.ArrayList; import java.util.Collections; @@ -194,25 +194,29 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes) { try { checkActive(); EntityId entityId = msg.getOriginator(); TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - RuleNodeId originatorNodeId = envelope.getOriginator(); List relations = nodeRoutes.get(originatorNodeId).stream() - .filter(r -> contains(envelope.getRelationTypes(), r.getType())) + .filter(r -> contains(relationTypes, r.getType())) .collect(Collectors.toList()); int relationsCount = relations.size(); if (relationsCount == 0) { log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId()); - if (envelope.getRelationTypes().contains(TbRelationTypes.FAILURE)) { - log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, envelope.getOriginator().getId()); + if (relationTypes.contains(TbRelationTypes.FAILURE)) { + log.debug("[{}] Failure during message processing by Rule Node [{}]. Enable and see debug events for more info", entityId, originatorNodeId.getId()); //TODO 2.5: Introduce our own RuleEngineFailureException to track what is wrong - msg.getCallback().onFailure(new RuntimeException("Failure during message processing by Rule Node [" + envelope.getOriginator().getId().toString() + "]")); + msg.getCallback().onFailure(new RuntimeException("Failure during message processing by Rule Node [" + originatorNodeId.getId().toString() + "]")); } else { msg.getCallback().onSuccess(); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7d567a31f1..7230804773 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -73,6 +73,10 @@ public final class TbMsg implements Serializable { data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); } + public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + } + private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); From 45bd764e6fec0f994f5345a20653c185f5b190ad Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 7 Apr 2020 11:45:28 +0300 Subject: [PATCH 043/196] Azure ServiceBus queue * created event hubs queue * created servicebus queue * refactored * refactored --- .../actors/ruleChain/DefaultTbContext.java | 2 +- .../RuleChainActorMessageProcessor.java | 2 +- .../src/main/resources/thingsboard.yml | 7 +- .../thingsboard/server/common/msg/TbMsg.java | 2 +- common/queue/pom.xml | 4 + .../azure/servicebus/TbServiceBusAdmin.java | 79 +++++++ .../TbServiceBusConsumerTemplate.java | 210 ++++++++++++++++++ .../TbServiceBusProducerTemplate.java | 110 +++++++++ .../servicebus/TbServiceBusSettings.java | 37 +++ .../queue/common/DefaultTbQueueMsg.java | 13 +- .../ConsistentHashPartitionService.java | 1 - .../provider/AwsSqsMonolithQueueFactory.java | 8 +- .../provider/AwsSqsTbCoreQueueFactory.java | 22 +- .../AwsSqsTbRuleEngineQueueFactory.java | 7 +- .../provider/AwsSqsTransportQueueFactory.java | 28 ++- .../InMemoryMonolithQueueFactory.java | 8 +- .../InMemoryTbTransportQueueFactory.java | 18 +- .../provider/KafkaMonolithQueueFactory.java | 8 +- .../provider/KafkaTbCoreQueueFactory.java | 6 +- .../KafkaTbRuleEngineQueueFactory.java | 4 +- .../provider/PubSubMonolithQueueFactory.java | 10 +- .../ServiceBusMonolithQueueFactory.java | 134 +++++++++++ .../ServiceBusTbCoreQueueProvider.java | 116 ++++++++++ .../ServiceBusTbRuleEngineQueueFactory.java | 99 +++++++++ .../ServiceBusTransportQueueFactory.java | 94 ++++++++ .../pubsub/TbPubSubConsumerTemplate.java | 11 +- .../pubsub/TbPubSubProducerTemplate.java | 3 +- .../queue/sqs/TbAwsSqsConsumerTemplate.java | 11 +- .../queue/sqs/TbAwsSqsProducerTemplate.java | 9 - pom.xml | 6 + .../src/main/resources/tb-mqtt-transport.yml | 7 +- 31 files changed, 978 insertions(+), 98 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 7178b94dfa..81444769d0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 3ab67bf81f..3a03cdadde 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 04c9205a12..cb562a7f52 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -517,7 +517,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -537,6 +537,11 @@ queue: ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7230804773..10c372d546 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 0a61eb0cf6..f55f69ece1 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -60,6 +60,10 @@ com.google.cloud google-cloud-pubsub + + com.microsoft.azure + azure-servicebus + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java new file mode 100644 index 0000000000..fb9cd89a8d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.microsoft.azure.servicebus.management.ManagementClient; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.TbQueueAdmin; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +public class TbServiceBusAdmin implements TbQueueAdmin { + + private final Set queues = ConcurrentHashMap.newKeySet(); + + private final ManagementClient client; + + public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { + ConnectionStringBuilder builder = new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + "queues", + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + + client = new ManagementClient(builder); + + try { + client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to get queues.", e); + throw new RuntimeException("Failed to get queues.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + if (queues.contains(topic)) { + return; + } + + try { + client.createQueue(topic); + queues.add(topic); + } catch (ServiceBusException | InterruptedException e) { + log.error("Failed to create queue: [{}]", topic, e); + } + } + + @PreDestroy + private void destroy() { + try { + client.close(); + } catch (IOException e) { + log.error("Failed to close ManagementClient."); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java new file mode 100644 index 0000000000..cca599d59a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.microsoft.azure.servicebus.TransactionContext; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; +import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; +import com.microsoft.azure.servicebus.primitives.MessagingEntityType; +import com.microsoft.azure.servicebus.primitives.MessagingFactory; +import com.microsoft.azure.servicebus.primitives.SettleModePair; +import lombok.extern.slf4j.Slf4j; +import org.apache.qpid.proton.amqp.messaging.Data; +import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; +import org.apache.qpid.proton.amqp.transport.SenderSettleMode; +import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class TbServiceBusConsumerTemplate implements TbQueueConsumer { + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbServiceBusSettings serviceBusSettings; + + private final Gson gson = new Gson(); + + private Set receivers; + private volatile Set partitions; + private volatile boolean subscribed; + private volatile boolean stopped = false; + private Map> pendingMessages = new ConcurrentHashMap<>(); + private volatile int messagesPerQueue; + + public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.serviceBusSettings = serviceBusSettings; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + receivers.forEach(CoreMessageReceiver::closeAsync); + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); + subscribed = true; + } + + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to parse message.", e); + throw new RuntimeException("Failed to parse message.", e); + } + }).collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + } + return Collections.emptyList(); + } + + private void createReceivers() { + List> receiverFutures = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .map(queue -> { + MessagingFactory factory; + try { + factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to create factory for the queue [{}]", queue); + throw new RuntimeException("Failed to create the factory", e); + } + + return CoreMessageReceiver.create(factory, queue, queue, 0, + new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), + MessagingEntityType.QUEUE); + }).collect(Collectors.toList()); + + try { + receivers = new HashSet<>(fromList(receiverFutures).get()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", topic); + } else { + log.error("Failed to create receivers", e); + } + } + } + + private ConnectionStringBuilder createConnection(String queue) { + admin.createTopicIfNotExists(queue); + return new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + queue, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + } + + private CompletableFuture> fromList(List> futures) { + CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; + futures.toArray(arrayFuture); + + return CompletableFuture + .allOf(arrayFuture) + .thenApply(v -> futures + .stream() + .map(CompletableFuture::join) + .collect(Collectors.toList())); + } + + @Override + public void commit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); + } + + private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java new file mode 100644 index 0000000000..5d9a931378 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java @@ -0,0 +1,110 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import com.google.gson.Gson; +import com.microsoft.azure.servicebus.IMessage; +import com.microsoft.azure.servicebus.Message; +import com.microsoft.azure.servicebus.QueueClient; +import com.microsoft.azure.servicebus.ReceiveMode; +import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.ServiceBusException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +public class TbServiceBusProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbServiceBusSettings serviceBusSettings; + private final Map clients = new HashMap<>(); + private ExecutorService executorService; + + public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.serviceBusSettings = serviceBusSettings; + executorService = Executors.newSingleThreadExecutor(); + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); + CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); + future.whenCompleteAsync((success, err) -> { + if (err != null) { + callback.onFailure(err); + } else { + callback.onSuccess(null); + } + }, executorService); + } + + @Override + public void stop() { + clients.forEach((t, client) -> { + try { + client.close(); + } catch (ServiceBusException e) { + log.error("Failed to close QueueClient.", e); + } + }); + + if (executorService != null) { + executorService.shutdownNow(); + } + } + + private QueueClient getClient(String topic) { + return clients.computeIfAbsent(topic, k -> { + admin.createTopicIfNotExists(topic); + ConnectionStringBuilder builder = + new ConnectionStringBuilder( + serviceBusSettings.getNamespaceName(), + topic, + serviceBusSettings.getSasKeyName(), + serviceBusSettings.getSasKey()); + try { + return new QueueClient(builder, ReceiveMode.PEEKLOCK); + } catch (InterruptedException | ServiceBusException e) { + log.error("Failed to create new client for the Queue: [{}]", topic, e); + throw new RuntimeException("Failed to create new client for the Queue", e); + } + }); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java new file mode 100644 index 0000000000..d872dcec0b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +@Component +@Data +public class TbServiceBusSettings { + @Value("${queue.service_bus.namespace_name}") + private String namespaceName; + @Value("${queue.service_bus.sas_key_name}") + private String sasKeyName; + @Value("${queue.service_bus.sas_key}") + private String sasKey; + @Value("${queue.service_bus.max_messages}") + private int maxMessages; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index 0e816ae59b..c7e439ef7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.common; -import com.google.gson.annotations.Expose; import lombok.Data; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgHeaders; @@ -26,12 +25,20 @@ import java.util.UUID; public class DefaultTbQueueMsg implements TbQueueMsg { private final UUID key; private final byte[] data; + private DefaultTbQueueMsgHeaders headers; + public DefaultTbQueueMsg(UUID key, byte[] data) { this.key = key; this.data = data; } - @Expose(serialize = false, deserialize = false) - private TbQueueMsgHeaders headers; + public DefaultTbQueueMsg(TbQueueMsg msg) { + this.key = msg.getKey(); + this.data = msg.getData(); + DefaultTbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); + msg.getHeaders().getData().forEach(headers::put); + this.headers = headers; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 1337a1cc74..f36ae70dfa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -36,7 +36,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 72d5731139..9f70345301 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -21,14 +21,14 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 1fba780395..770e7fa65c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -18,21 +18,22 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -48,8 +49,6 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - - private final TbQueueAdmin admin; public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, @@ -78,7 +77,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); } @@ -88,7 +87,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } @@ -99,15 +98,16 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 4181a9442d..f87f108453 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -24,12 +24,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; @@ -86,7 +86,8 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 09ca194724..4196b226ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -18,16 +18,20 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -55,17 +59,17 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbAwsSqsProducerTemplate> producerTemplate = + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbAwsSqsProducerTemplate> producerTemplate = new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); - TbAwsSqsConsumerTemplate> consumerTemplate = + TbAwsSqsConsumerTemplate> consumerTemplate = new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); templateBuilder.queueAdmin(admin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); @@ -76,18 +80,18 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { + public TbQueueProducer> createTbCoreMsgProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { + public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index b40876d661..a23c93e015 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -26,14 +26,14 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Slf4j diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 9914cf65f9..c779127e1f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -19,22 +19,22 @@ import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 8b84d08bff..fa36f8284b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -26,17 +26,17 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 7083553abb..5e46d67eed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -26,16 +26,16 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 9b8a9c4ba0..eef10f35dc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -24,15 +24,15 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 4aa35e5d68..f00bd7caef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -27,11 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,6 +35,10 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @@ -54,8 +54,6 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private TbQueueProducer> tbCoreProducer; - public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java new file mode 100644 index 0000000000..914e200fc9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") +public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java new file mode 100644 index 0000000000..334af58133 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") +public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { + + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin admin; + + public ServiceBusTbCoreQueueProvider(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.serviceBusSettings = serviceBusSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..3ceb837606 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") +public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + + public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbServiceBusSettings serviceBusSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java new file mode 100644 index 0000000000..d38a4389a3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + + public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceBusSettings = serviceBusSettings; + this.admin = admin; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbQueueProducer> producerTemplate = + new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + + TbQueueConsumer> consumerTemplate = + new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 4109e72070..5dc795739a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -20,7 +20,6 @@ import com.google.api.core.ApiFutures; import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; import com.google.cloud.pubsub.v1.stub.SubscriberStub; import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import com.google.pubsub.v1.AcknowledgeRequest; @@ -36,15 +35,12 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -139,7 +135,7 @@ public class TbPubSubConsumerTemplate implements TbQueueCo subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); subscriptionNames.forEach(admin::createTopicIfNotExists); consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages()/subscriptionNames.size(); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); subscribed = true; } List messages; @@ -217,11 +213,6 @@ public class TbPubSubConsumerTemplate implements TbQueueCo public T decode(PubsubMessage message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); - TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - Map headerMap = gson.fromJson(message.getAttributesMap().get("headers"), new TypeToken>() { - }.getType()); - headerMap.forEach(headers::put); - msg.setHeaders(headers); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index fb9bc8b189..2cd2e1054e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -71,7 +71,6 @@ public class TbPubSubProducerTemplate implements TbQueuePr public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); pubsubMessageBuilder.setData(getMsg(msg)); - pubsubMessageBuilder.putAttributes("headers", gson.toJson(msg.getHeaders().getData())); Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); @@ -110,7 +109,7 @@ public class TbPubSubProducerTemplate implements TbQueuePr } private ByteString getMsg(T msg) { - String json = gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData())); + String json = gson.toJson(new DefaultTbQueueMsg(msg)); return ByteString.copyFrom(json.getBytes()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index b4a2f84b83..3e71388844 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -23,7 +23,6 @@ import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; import com.amazonaws.services.sqs.model.Message; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; -import com.google.common.reflect.TypeToken; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -38,15 +37,12 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -161,7 +157,7 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo if (stopped) { log.info("[{}] Aws SQS consumer is stopped.", topic); } else { - log.error("Failed to pool messages.", e); + log.error("Failed to pool messages.", e); } } } @@ -214,11 +210,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo public T decode(Message message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); - Map headerMap = gson.fromJson(message.getMessageAttributes().get("headers").getStringValue(), new TypeToken>() { - }.getType()); - headerMap.forEach(headers::put); - msg.setHeaders(headers); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index a5707c845d..d304953ef4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -20,7 +20,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.MessageAttributeValue; import com.amazonaws.services.sqs.model.SendMessageRequest; import com.amazonaws.services.sqs.model.SendMessageResult; import com.google.common.util.concurrent.FutureCallback; @@ -37,7 +36,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -82,13 +80,6 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData()))); - Map attributes = new HashMap<>(); - - attributes.put("headers", new MessageAttributeValue() - .withStringValue(gson.toJson(msg.getHeaders().getData())) - .withDataType("String")); - - sendMsgRequest.withMessageAttributes(attributes); sendMsgRequest.withMessageGroupId(msg.getKey().toString()); ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); diff --git a/pom.xml b/pom.xml index e385bff9ce..96f336c3a2 100755 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ 1.23 1.11.747 1.84.0 + 3.2.0 1.5.0 1.4.3 @@ -898,6 +899,11 @@ google-cloud-pubsub ${pubsub.client.version} + + com.microsoft.azure + azure-servicebus + ${azure-servicebus.version} + org.passay passay diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index a875d56782..37ac4c14c2 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -83,6 +83,11 @@ queue: ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 11479935c2d511288c04aaca87a70e8311388ab4 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Tue, 7 Apr 2020 11:55:26 +0300 Subject: [PATCH 044/196] SQL TTL Implemenation * init commit * improvements for ttl-functions * added drop partitions by ttl function * add load ttl function to upgrade script * fix typo * add IF NOT EXISTS for json_v in timescale upgrade * changed logic for removing customer records from ts_kv * improvements in upgrade scenario * improvements * added intial delay for TTL task execution --- .../schema_update_psql_drop_partitions.sql | 123 ++++++++ .../data/upgrade/2.4.3/schema_update_ttl.sql | 125 ++++++++ .../AbstractSqlTsDatabaseUpgradeService.java | 24 +- .../install/PsqlTsDatabaseUpgradeService.java | 66 +++-- .../TimescaleTsDatabaseUpgradeService.java | 54 ++-- .../ttl/AbstractTimeseriesCleanUpService.java | 89 ++++++ .../ttl/PsqlTimeseriesCleanUpService.java | 41 +++ .../TimescaleTimeseriesCleanUpService.java | 35 +++ .../src/main/resources/thingsboard.yml | 4 + .../server/dao/util/PsqlTsDao.java | 21 ++ .../main/resources/sql/schema-timescale.sql | 120 +++++++- dao/src/main/resources/sql/schema-ts-psql.sql | 271 ++++++++++++++++-- 12 files changed, 899 insertions(+), 74 deletions(-) create mode 100644 application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql create mode 100644 application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql new file mode 100644 index 0000000000..0916c241a1 --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql @@ -0,0 +1,123 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + max_tenant_ttl bigint; + max_customer_ttl bigint; + max_ttl bigint; + date timestamp; + partition_by_max_ttl_date varchar; + partition_month varchar; + partition_day varchar; + partition_year varchar; + partition varchar; + partition_to_delete varchar; + + +BEGIN + SELECT max(attribute_kv.long_v) + FROM tenant + INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_tenant_ttl; + SELECT max(attribute_kv.long_v) + FROM customer + INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_customer_ttl; + max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); + if max_ttl IS NOT NULL AND max_ttl > 0 THEN + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000)); + partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); + RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; + IF partition_by_max_ttl_date IS NOT NULL THEN + CASE + WHEN partition_type = 'DAYS' THEN + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5); + WHEN partition_type = 'MONTHS' THEN + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + ELSE + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + END CASE; + FOR partition IN SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename like 'ts_kv_' || '%' + AND tablename != 'ts_kv_latest' + AND tablename != 'ts_kv_dictionary' + LOOP + IF partition != partition_by_max_ttl_date THEN + IF partition_year IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN + partition_to_delete := partition; + ELSE + IF partition_month IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN + partition_to_delete := partition; + ELSE + IF partition_day IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN + partition_to_delete := partition; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + IF partition_to_delete IS NOT NULL THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; + EXECUTE format('DROP TABLE %I', partition_to_delete); + deleted := deleted + 1; + END IF; + END LOOP; + END IF; + END IF; +END +$$; + +CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS +$$ +BEGIN + CASE + WHEN partition_type = 'DAYS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd'); + WHEN partition_type = 'MONTHS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM'); + WHEN partition_type = 'YEARS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy'); + WHEN partition_type = 'INDEFINITE' THEN + partition := NULL; + ELSE + partition := NULL; + END CASE; + IF partition IS NOT NULL THEN + IF NOT EXISTS(SELECT + FROM pg_tables + WHERE schemaname = 'public' + AND tablename = partition) THEN + partition := NULL; + RAISE NOTICE 'Failed to found partition by ttl'; + END IF; + END IF; +END; +$$ LANGUAGE plpgsql; diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql new file mode 100644 index 0000000000..ff1fb5129b --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -0,0 +1,125 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31), + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record varchar; + customer_id_record varchar; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java index 70e56f9489..ab03ca23ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java @@ -47,7 +47,7 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { @Autowired protected InstallScripts installScripts; - protected abstract void loadSql(Connection conn); + protected abstract void loadSql(Connection conn, String fileName); protected void loadFunctions(Path sqlFile, Connection conn) throws Exception { String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); @@ -70,6 +70,26 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { return versionValid; } + protected boolean isOldSchema(Connection conn, long fromVersion) { + boolean isOldSchema = true; + try { + Statement statement = conn.createStatement(); + statement.execute("CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version));"); + Thread.sleep(1000); + ResultSet resultSet = statement.executeQuery("SELECT schema_version FROM tb_schema_settings;"); + if (resultSet.next()) { + isOldSchema = resultSet.getLong(1) <= fromVersion; + } else { + resultSet.close(); + statement.execute("INSERT INTO tb_schema_settings (schema_version) VALUES (" + fromVersion + ")"); + } + statement.close(); + } catch (InterruptedException | SQLException e) { + log.info("Failed to check current PostgreSQL schema due to: {}", e.getMessage()); + } + return isOldSchema; + } + protected void executeQuery(Connection conn, String query) { try { Statement statement = conn.createStatement(); @@ -83,7 +103,7 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { nextWarning = nextWarning.getNextWarning(); } } - Thread.sleep(5000); + Thread.sleep(2000); log.info("Successfully executed query: {}", query); } catch (InterruptedException | SQLException e) { log.info("Failed to execute query: {} due to: {}", query, e.getMessage()); diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index a630540981..5f97a6eaa5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -34,6 +34,8 @@ import java.sql.DriverManager; public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; + private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; + private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql"; private static final String TS_KV_OLD = "ts_kv_old;"; private static final String TS_KV_LATEST_OLD = "ts_kv_latest_old;"; @@ -76,30 +78,39 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!"); } else { log.info("PostgreSQL version is valid!"); - log.info("Load upgrade functions ..."); - loadSql(conn); - log.info("Updating timeseries schema ..."); - executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE); - executeQuery(conn, CALL_CREATE_PARTITIONS); - executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); - executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); - executeQuery(conn, CALL_INSERT_INTO_TS_KV); - executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); - executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); - - executeQuery(conn, DROP_TABLE_TS_KV_OLD); - executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD); - - executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE); - executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITIONS); - executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV); - executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); - - executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN json_v json;"); - executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;"); + if (isOldSchema(conn, 2004003)) { + log.info("Load upgrade functions ..."); + loadSql(conn, LOAD_FUNCTIONS_SQL); + log.info("Updating timeseries schema ..."); + executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE); + executeQuery(conn, CALL_CREATE_PARTITIONS); + executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); + executeQuery(conn, CALL_INSERT_INTO_TS_KV); + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); + + executeQuery(conn, DROP_TABLE_TS_KV_OLD); + executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD); + + executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE); + executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITIONS); + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV); + executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); + + executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); + executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;"); + } + + log.info("Load TTL functions ..."); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + log.info("Load Drop Partitions functions ..."); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000"); log.info("schema timeseries updated!"); } @@ -110,11 +121,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe } } - protected void loadSql(Connection conn) { - Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); + @Override + protected void loadSql(Connection conn, String fileName) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName); try { loadFunctions(schemaUpdateFile, conn); - log.info("Upgrade functions successfully loaded!"); + log.info("Functions successfully loaded!"); } catch (Exception e) { log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 99afd59a50..21bdb500d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -39,6 +39,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr private long chunkTimeInterval; private static final String LOAD_FUNCTIONS_SQL = "schema_update_timescale_ts.sql"; + private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; private static final String TENANT_TS_KV_OLD_TABLE = "tenant_ts_kv_old;"; @@ -79,31 +80,37 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!"); } else { log.info("PostgreSQL version is valid!"); - log.info("Load upgrade functions ..."); - loadSql(conn); - log.info("Updating timescale schema ..."); - executeQuery(conn, CALL_CREATE_TS_KV_LATEST_TABLE); - executeQuery(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE); + if (isOldSchema(conn, 2004003)) { + log.info("Load upgrade functions ..."); + loadSql(conn, LOAD_FUNCTIONS_SQL); + log.info("Updating timescale schema ..."); + executeQuery(conn, CALL_CREATE_TS_KV_LATEST_TABLE); + executeQuery(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE); - executeQuery(conn, "SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); + executeQuery(conn, "SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); - executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); - executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); - executeQuery(conn, CALL_INSERT_INTO_TS_KV); - executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); + executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); + executeQuery(conn, CALL_INSERT_INTO_TS_KV); + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); - executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE); + executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE); - executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE); - executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY); - executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE); + executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY); + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); - executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN json_v json;"); - executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;"); + executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); + executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;"); + } + log.info("Load TTL functions ..."); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000"); log.info("schema timescale updated!"); } } @@ -113,13 +120,14 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr } } - protected void loadSql(Connection conn) { - Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); + @Override + protected void loadSql(Connection conn, String fileName) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName); try { loadFunctions(schemaUpdateFile, conn); - log.info("Upgrade functions successfully loaded!"); + log.info("Functions successfully loaded!"); } catch (Exception e) { - log.info("Failed to load Timescale upgrade functions due to: {}", e.getMessage()); + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); } } } \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java new file mode 100644 index 0000000000..2399450d86 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.Statement; + +@PsqlTsAnyDao +@Slf4j +public abstract class AbstractTimeseriesCleanUpService { + + @Value("${sql.ttl.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Value("${spring.datasource.url}") + private String dbUrl; + + @Value("${spring.datasource.username}") + private String dbUserName; + + @Value("${spring.datasource.password}") + private String dbPassword; + + @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + protected abstract void doCleanUp(Connection connection); + + protected long executeQuery(Connection conn, String query) { + long removed = 0L; + try { + Statement statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery(query); + getWarnings(statement); + resultSet.next(); + removed = resultSet.getLong(1); + log.debug("Successfully executed query: {}", query); + } catch (SQLException e) { + log.debug("Failed to execute query: {} due to: {}", query, e.getMessage()); + } + return removed; + } + + private void getWarnings(Statement statement) throws SQLException { + SQLWarning warnings = statement.getWarnings(); + if (warnings != null) { + log.debug("{}", warnings.getMessage()); + SQLWarning nextWarning = warnings.getNextWarning(); + while (nextWarning != null) { + log.debug("{}", nextWarning.getMessage()); + nextWarning = nextWarning.getNextWarning(); + } + } + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java new file mode 100644 index 0000000000..ab344dc518 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.util.PsqlTsDao; + +import java.sql.Connection; + +@PsqlTsDao +@Service +@Slf4j +public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService { + + @Value("${sql.postgres.ts_key_value_partitioning}") + private String partitionType; + + @Override + protected void doCleanUp(Connection connection) { + long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); + log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); + long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);"); + log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java new file mode 100644 index 0000000000..1dbdb4ad55 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.util.TimescaleDBTsDao; + +import java.sql.Connection; + +@TimescaleDBTsDao +@Service +@Slf4j +public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService { + + @Override + protected void doCleanUp(Connection connection) { + long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);"); + log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); + } +} \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 820fe14be1..0402c91387 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -216,6 +216,10 @@ sql: timescale: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" + ttl: + enabled: "${SQL_TTL_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds + ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java new file mode 100644 index 0000000000..cc0d9051e5 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("'${database.ts.type}'=='sql' && '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") +public @interface PsqlTsDao { } \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index b95c8b86ba..32e2a78620 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -44,4 +44,122 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( dbl_v double precision, json_v json, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); \ No newline at end of file +); + +CREATE TABLE IF NOT EXISTS tb_schema_settings +( + schema_version bigint NOT NULL, + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) +); + +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31), + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record varchar; + customer_id_record varchar; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 32b6762c8e..3789444106 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -14,31 +14,260 @@ -- limitations under the License. -- -CREATE TABLE IF NOT EXISTS ts_kv ( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json +CREATE TABLE IF NOT EXISTS ts_kv +( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json ) PARTITION BY RANGE (ts); -CREATE TABLE IF NOT EXISTS ts_kv_latest ( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, +CREATE TABLE IF NOT EXISTS ts_kv_latest +( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( - key varchar(255) NOT NULL, +CREATE TABLE IF NOT EXISTS ts_kv_dictionary +( + key varchar(255) NOT NULL, key_id serial UNIQUE, CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) -); \ No newline at end of file +); + +CREATE TABLE IF NOT EXISTS tb_schema_settings +( + schema_version bigint NOT NULL, + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) +); + +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); + +CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + max_tenant_ttl bigint; + max_customer_ttl bigint; + max_ttl bigint; + date timestamp; + partition_by_max_ttl_date varchar; + partition_month varchar; + partition_day varchar; + partition_year varchar; + partition varchar; + partition_to_delete varchar; + + +BEGIN + SELECT max(attribute_kv.long_v) + FROM tenant + INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_tenant_ttl; + SELECT max(attribute_kv.long_v) + FROM customer + INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_customer_ttl; + max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); + if max_ttl IS NOT NULL AND max_ttl > 0 THEN + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000)); + partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); + RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; + IF partition_by_max_ttl_date IS NOT NULL THEN + CASE + WHEN partition_type = 'DAYS' THEN + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5); + WHEN partition_type = 'MONTHS' THEN + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + ELSE + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + END CASE; + FOR partition IN SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename like 'ts_kv_' || '%' + AND tablename != 'ts_kv_latest' + AND tablename != 'ts_kv_dictionary' + LOOP + IF partition != partition_by_max_ttl_date THEN + IF partition_year IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN + partition_to_delete := partition; + ELSE + IF partition_month IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN + partition_to_delete := partition; + ELSE + IF partition_day IS NOT NULL THEN + IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN + partition_to_delete := partition; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + IF partition_to_delete IS NOT NULL THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; + EXECUTE format('DROP TABLE %I', partition_to_delete); + deleted := deleted + 1; + END IF; + END LOOP; + END IF; + END IF; +END +$$; + +CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS +$$ +BEGIN + CASE + WHEN partition_type = 'DAYS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd'); + WHEN partition_type = 'MONTHS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM'); + WHEN partition_type = 'YEARS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy'); + WHEN partition_type = 'INDEFINITE' THEN + partition := NULL; + ELSE + partition := NULL; + END CASE; + IF partition IS NOT NULL THEN + IF NOT EXISTS(SELECT + FROM pg_tables + WHERE schemaname = 'public' + AND tablename = partition) THEN + partition := NULL; + RAISE NOTICE 'Failed to found partition by ttl'; + END IF; + END IF; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31), + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record varchar; + customer_id_record varchar; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; From 3eaae1ef32cf601ca06001e95a5f5f130c4d74a8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 8 Apr 2020 14:09:56 +0300 Subject: [PATCH 045/196] Statistics Implementation --- .../server/actors/app/AppActor.java | 35 +---- .../device/DeviceActorMessageProcessor.java | 4 +- .../actors/ruleChain/DefaultTbContext.java | 39 +++--- .../RemoteToRuleChainTellNextMsg.java | 48 ------- .../actors/ruleChain/RuleChainActor.java | 16 +-- .../RuleChainActorMessageProcessor.java | 34 +++-- .../ruleChain/RuleChainManagerActor.java | 68 ++++++++-- .../RuleNodeToRuleChainTellNextMsg.java | 1 + .../actors/shared/EntityActorsManager.java | 93 ------------- .../shared/rulechain/RuleChainManager.java | 59 -------- .../rulechain/SystemRuleChainManager.java | 48 ------- .../rulechain/TenantRuleChainManager.java | 53 -------- .../server/actors/stats/StatsActor.java | 1 - .../server/actors/tenant/TenantActor.java | 66 +++++---- .../controller/plugin/TbWebSocketHandler.java | 6 +- .../queue/DefaultTbCoreConsumerService.java | 20 +-- .../DefaultTbRuleEngineConsumerService.java | 54 +++++--- .../service/queue/TbMsgPackCallback.java | 78 +++++++++++ ...gPackCallback.java => TbPackCallback.java} | 21 +-- .../queue/TbRuleEngineConsumerStats.java | 119 +++++++++++++--- .../queue/TbTenantRuleEngineStats.java | 92 +++++++++++++ .../processing/AbstractConsumerService.java | 10 +- .../TbRuleEngineProcessingResult.java | 18 ++- ...TbRuleEngineProcessingStrategyFactory.java | 31 +++-- .../service/rpc/FromDeviceRpcResponse.java | 1 - .../rpc/ToDeviceRpcRequestActorMsg.java | 3 - .../state/DefaultDeviceStateService.java | 4 +- .../service/state/DeviceStateService.java | 4 +- .../DefaultRuleEngineStatisticsService.java | 127 ++++++++++++++++++ .../stats/RuleEngineStatisticsService.java | 12 +- .../DefaultSubscriptionManagerService.java | 12 +- .../DefaultTbLocalSubscriptionService.java | 8 +- .../SubscriptionManagerService.java | 10 +- .../TbLocalSubscriptionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 6 +- .../TelemetrySubscriptionService.java | 4 - .../service/telemetry/sub/Subscription.java | 80 ----------- .../BaseRuleChainTransactionService.java | 13 -- .../msg/TransportToDeviceActorMsgWrapper.java | 6 +- application/src/main/resources/logback.xml | 4 +- .../src/main/resources/thingsboard.yml | 6 +- ...AbstractRuleEngineFlowIntegrationTest.java | 2 - ...actRuleEngineLifecycleIntegrationTest.java | 2 - .../server/common/msg/MsgType.java | 5 - .../thingsboard/server/common/msg/TbMsg.java | 4 +- .../common/msg/cluster/ServerAddress.java | 47 ------- .../msg/queue/QueueToRuleEngineMsg.java | 1 + .../common/msg/queue/RuleEngineException.java | 38 ++++++ .../common/msg/queue/RuleNodeException.java | 58 ++++++++ .../server/common/msg/queue/TbCallback.java | 37 +++++ .../common/msg/queue/TbMsgCallback.java | 4 +- .../MultipleTbQueueTbMsgCallbackWrapper.java | 4 +- .../common/TbQueueTbMsgCallbackWrapper.java | 6 +- .../ConsistentHashPartitionService.java | 37 ++--- .../DefaultTbServiceInfoProvider.java | 13 ++ .../queue/discovery/PartitionService.java | 2 + .../discovery/TbServiceInfoProvider.java | 8 ++ .../settings/TbQueueRuleEngineSettings.java | 3 + common/queue/src/main/proto/queue.proto | 1 + .../api/RuleChainTransactionService.java | 3 - .../rule/engine/api/TbContext.java | 4 +- .../rule/engine/flow/TbAckNode.java | 53 ++++++++ .../rule/engine/flow/TbCheckpointNode.java | 57 ++++++++ .../flow/TbCheckpointNodeConfiguration.java | 24 ++-- .../rule/engine/rpc/TbSendRPCRequestNode.java | 3 +- .../src/main/resources/tb-mqtt-transport.yml | 38 +++++- 66 files changed, 1014 insertions(+), 758 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java delete mode 100644 application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java rename application/src/main/java/org/thingsboard/server/service/queue/{MsgPackCallback.java => TbPackCallback.java} (70%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java create mode 100644 application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java => application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java (72%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java rename common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java => rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java (53%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index c096238a8b..145ae10d06 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -24,10 +24,9 @@ import akka.actor.Terminated; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; +import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager; import org.thingsboard.server.actors.tenant.TenantActor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; @@ -37,16 +36,14 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; -public class AppActor extends RuleChainManagerActor { +public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; @@ -54,7 +51,7 @@ public class AppActor extends RuleChainManagerActor { private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { - super(systemContext, new SystemRuleChainManager(systemContext)); + super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); } @@ -80,9 +77,6 @@ public class AppActor extends RuleChainManagerActor { switch (msg.getMsgType()) { case APP_INIT_MSG: break; - case SEND_TO_CLUSTER_MSG: - onPossibleClusterMsg((SendToClusterMsg) msg); - break; case PARTITION_CHANGE_MSG: broadcast(msg); break; @@ -98,7 +92,6 @@ public class AppActor extends RuleChainManagerActor { case DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG: case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onToDeviceActorMsg((TenantAwareMsg) msg); break; default: @@ -110,7 +103,6 @@ public class AppActor extends RuleChainManagerActor { private void initRuleChainsAndTenantActors() { log.info("Starting main system actor."); try { - initRuleChains(); if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); for (Tenant tenant : tenantIterator) { @@ -125,37 +117,22 @@ public class AppActor extends RuleChainManagerActor { } } - private void onPossibleClusterMsg(SendToClusterMsg msg) { - //TODO 2.5 -// Optional address = systemContext.getRoutingService().resolveById(msg.getEntityId()); -// if (address.isPresent()) { - -// systemContext.getRpcService().tell( -// systemContext.getEncodingService().convertToProtoDataMessage(address.get(), msg.getMsg())); -// } else { - self().tell(msg.getMsg(), ActorRef.noSender()); -// } - } - private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { if (SYSTEM_TENANT.equals(msg.getTenantId())) { -// this may be a notification about system entities created. -// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg); + msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); } } - @Override protected void broadcast(Object msg) { - super.broadcast(msg); tenantActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { ActorRef target = null; if (SYSTEM_TENANT.equals(msg.getTenantId())) { - target = getEntityActorRef(msg.getEntityId()); + log.warn("Message has system tenant id: {}", msg); } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 307797f864..83d269ef6d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -56,7 +56,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; @@ -213,7 +213,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { void process(ActorContext context, TransportToDeviceActorMsgWrapper wrapper) { TransportToDeviceActorMsg msg = wrapper.getMsg(); - TbMsgCallback callback = wrapper.getCallback(); + TbCallback callback = wrapper.getCallback(); if (msg.hasSessionEvent()) { processSessionStateMsgs(msg.getSessionInfo(), msg.getSessionEvent()); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 7178b94dfa..0e8033fac2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -110,7 +110,7 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { relationTypes.forEach(relationType -> mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, relationType, th)); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), relationTypes, msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } @Override @@ -139,53 +139,61 @@ class DefaultTbContext implements TbContext { mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); } + @Override + public void enqueueForTellFailure(TbMsg tbMsg, String failureMessage) { + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(TbRelationTypes.FAILURE), failureMessage, null, null); + } + @Override public void enqueueForTellNext(TbMsg tbMsg, String relationType) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, null, null); } @Override public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, null, null); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, null, null); } @Override public void enqueueForTellNext(TbMsg tbMsg, String relationType, Runnable onSuccess, Consumer onFailure) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); } @Override public void enqueueForTellNext(TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); } @Override public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), onSuccess, onFailure); + TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); + enqueueForTellNext(tpi, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure); } @Override public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set relationTypes, Runnable onSuccess, Consumer onFailure) { TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator()); - enqueueForTellNext(tpi, tbMsg, relationTypes, onSuccess, onFailure); + enqueueForTellNext(tpi, tbMsg, relationTypes, null, onSuccess, onFailure); } - private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, Runnable onSuccess, Consumer onFailure) { + private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg tbMsg, Set relationTypes, String failureMessage, Runnable onSuccess, Consumer onFailure) { RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId(); RuleNodeId ruleNodeId = nodeCtx.getSelf().getId(); tbMsg = TbMsg.newMsg(tbMsg, ruleChainId, ruleNodeId); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(tbMsg)) - .addAllRelationTypes(relationTypes) - .build(); - mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + .addAllRelationTypes(relationTypes); + if (failureMessage != null) { + msg.setFailureMessage(failureMessage); + } + mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg.build()), new SimpleTbQueueCallback(onSuccess, onFailure)); } @Override @@ -207,7 +215,8 @@ class DefaultTbContext implements TbContext { if (nodeCtx.getSelf().isDebugMode()) { mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), msg, TbRelationTypes.FAILURE, th); } - nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), msg), nodeCtx.getSelfActor()); + nodeCtx.getChainActor().tell(new RuleNodeToRuleChainTellNextMsg(nodeCtx.getSelf().getId(), Collections.singleton(TbRelationTypes.FAILURE), + msg, th != null ? th.getMessage() : null), nodeCtx.getSelfActor()); } public void updateSelf(RuleNode self) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java deleted file mode 100644 index d062c5dc49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RemoteToRuleChainTellNextMsg.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.ruleChain; - -import lombok.Data; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; -import org.thingsboard.server.common.msg.aware.TenantAwareMsg; - -import java.io.Serializable; - -/** - * Created by ashvayka on 19.03.18. - */ -@Data -final class RemoteToRuleChainTellNextMsg extends RuleNodeToRuleChainTellNextMsg implements TenantAwareMsg, RuleChainAwareMsg { - - private static final long serialVersionUID = 2459605482321657447L; - private final TenantId tenantId; - private final RuleChainId ruleChainId; - - public RemoteToRuleChainTellNextMsg(RuleNodeToRuleChainTellNextMsg original, TenantId tenantId, RuleChainId ruleChainId) { - super(original.getOriginator(), original.getRelationTypes(), original.getMsg()); - this.tenantId = tenantId; - this.ruleChainId = ruleChainId; - } - - @Override - public MsgType getMsgType() { - return MsgType.REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java index 015e7a008b..027f4aac9d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.service.ComponentActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @@ -30,9 +31,9 @@ import scala.concurrent.duration.Duration; public class RuleChainActor extends ComponentActor { - private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) { - super(systemContext, tenantId, ruleChainId); - setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext, + private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChain ruleChain) { + super(systemContext, tenantId, ruleChain.getId()); + setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChain, systemContext, context().parent(), context().self())); } @@ -46,7 +47,6 @@ public class RuleChainActor extends ComponentActor> nodeRoutes; private final RuleChainService service; private final TbQueueProducer> producer; + private String ruleChainName; private RuleNodeId firstId; private RuleNodeCtx firstNode; private boolean started; - RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { - super(systemContext, tenantId, ruleChainId); + super(systemContext, tenantId, ruleChain.getId()); + this.ruleChainName = ruleChain.getName(); this.parent = parent; this.self = self; this.nodeActors = new HashMap<>(); @@ -113,6 +117,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor ruleNodeList = service.getRuleChainNodes(tenantId, entityId); log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size()); for (RuleNode ruleNode : ruleNodeList) { @@ -194,7 +199,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes) { + private void onTellNext(TbMsg msg, RuleNodeId originatorNodeId, Set relationTypes, String failureMessage) { try { checkActive(); EntityId entityId = msg.getOriginator(); @@ -245,9 +250,14 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor actors; + @Getter + protected RuleChain rootChain; + @Getter + protected ActorRef rootChainActor; - public RuleChainManagerActor(ActorSystemContext systemContext, RuleChainManager ruleChainManager) { + public RuleChainManagerActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.ruleChainManager = ruleChainManager; + this.tenantId = tenantId; + this.actors = HashBiMap.create(); this.ruleChainService = systemContext.getRuleChainService(); } protected void initRuleChains() { - ruleChainManager.init(this.context()); + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) { + RuleChainId ruleChainId = ruleChain.getId(); + log.debug("[{}|{}] Creating rule chain actor", ruleChainId.getEntityType(), ruleChain.getId()); + //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. + ActorRef actorRef = getOrCreateActor(this.context(), ruleChainId, id -> ruleChain); + visit(ruleChain, actorRef); + log.debug("[{}|{}] Rule Chain actor created.", ruleChainId.getEntityType(), ruleChainId.getId()); + } + } + + protected void visit(RuleChain entity, ActorRef actorRef) { + if (entity != null && entity.isRoot()) { + rootChain = entity; + rootChainActor = actorRef; + } + } + + public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId) { + return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); + } + + public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId, Function provider) { + return actors.computeIfAbsent(ruleChainId, eId -> { + RuleChain ruleChain = provider.apply(eId); + return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) + .withDispatcher(DefaultActorService.TENANT_RULE_DISPATCHER_NAME), eId.toString()); + }); } protected ActorRef getEntityActorRef(EntityId entityId) { ActorRef target = null; - switch (entityId.getEntityType()) { - case RULE_CHAIN: - target = ruleChainManager.getOrCreateActor(this.context(), (RuleChainId) entityId); - break; + if (entityId.getEntityType() == EntityType.RULE_CHAIN) { + target = getOrCreateActor(this.context(), (RuleChainId) entityId); } return target; } protected void broadcast(Object msg) { - ruleChainManager.broadcast(msg); + actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); } + + public ActorRef get(RuleChainId id) { + return actors.get(id); + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java index 1fda91caf0..2bccc654e0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeToRuleChainTellNextMsg.java @@ -34,6 +34,7 @@ class RuleNodeToRuleChainTellNextMsg implements TbActorMsg, Serializable { private final RuleNodeId originator; private final Set relationTypes; private final TbMsg msg; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java deleted file mode 100644 index e45f69ca9e..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/EntityActorsManager.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared; - -import akka.actor.ActorContext; -import akka.actor.ActorRef; -import akka.actor.Props; -import akka.actor.UntypedActor; -import akka.japi.Creator; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.service.ContextAwareActor; -import org.thingsboard.server.common.data.SearchTextBased; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.page.PageDataIterable; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class EntityActorsManager> { - - protected final ActorSystemContext systemContext; - protected final BiMap actors; - - public EntityActorsManager(ActorSystemContext systemContext) { - this.systemContext = systemContext; - this.actors = HashBiMap.create(); - } - - protected abstract TenantId getTenantId(); - - protected abstract String getDispatcherName(); - - protected abstract Creator creator(T entityId); - - protected abstract PageDataIterable.FetchFunction getFetchEntitiesFunction(); - - public void init(ActorContext context) { - for (M entity : new PageDataIterable<>(getFetchEntitiesFunction(), ContextAwareActor.ENTITY_PACK_LIMIT)) { - T entityId = (T) entity.getId(); - log.debug("[{}|{}] Creating entity actor", entityId.getEntityType(), entityId.getId()); - //TODO: remove this cast making UUIDBased subclass of EntityId an interface and vice versa. - ActorRef actorRef = getOrCreateActor(context, entityId); - visit(entity, actorRef); - log.debug("[{}|{}] Entity actor created.", entityId.getEntityType(), entityId.getId()); - } - } - - public void visit(M entity, ActorRef actorRef) { - } - - public ActorRef getOrCreateActor(ActorContext context, T entityId) { - return actors.computeIfAbsent(entityId, eId -> - context.actorOf(Props.create(creator(eId)) - .withDispatcher(getDispatcherName()), eId.toString())); - } - - public void broadcast(Object msg) { - actors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender())); - } - - public void remove(T id) { - actors.remove(id); - } - - public ActorRef get(T id) { - return actors.get(id); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java deleted file mode 100644 index ef2b4282d4..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/RuleChainManager.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorRef; -import akka.japi.Creator; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.ruleChain.RuleChainActor; -import org.thingsboard.server.actors.shared.EntityActorsManager; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.rule.RuleChainService; - -/** - * Created by ashvayka on 15.03.18. - */ -@Slf4j -public abstract class RuleChainManager extends EntityActorsManager { - - protected final RuleChainService service; - @Getter - protected RuleChain rootChain; - @Getter - protected ActorRef rootChainActor; - - public RuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - this.service = systemContext.getRuleChainService(); - } - - @Override - public Creator creator(RuleChainId entityId) { - return new RuleChainActor.ActorCreator(systemContext, getTenantId(), entityId); - } - - @Override - public void visit(RuleChain entity, ActorRef actorRef) { - if (entity != null && entity.isRoot()) { - rootChain = entity; - rootChainActor = actorRef; - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java deleted file mode 100644 index f2a80a9e82..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/SystemRuleChainManager.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.page.TextPageData; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.dao.model.ModelConstants; - -import java.util.Collections; - -public class SystemRuleChainManager extends RuleChainManager { - - public SystemRuleChainManager(ActorSystemContext systemContext) { - super(systemContext); - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> new TextPageData<>(Collections.emptyList(), link); - } - - @Override - protected TenantId getTenantId() { - return ModelConstants.SYSTEM_TENANT; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME; - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java b/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java deleted file mode 100644 index cfe862af49..0000000000 --- a/application/src/main/java/org/thingsboard/server/actors/shared/rulechain/TenantRuleChainManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.actors.shared.rulechain; - -import akka.actor.ActorContext; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable.FetchFunction; -import org.thingsboard.server.common.data.rule.RuleChain; - -public class TenantRuleChainManager extends RuleChainManager { - - private final TenantId tenantId; - - public TenantRuleChainManager(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext); - this.tenantId = tenantId; - } - - @Override - public void init(ActorContext context) { - super.init(context); - } - - @Override - protected TenantId getTenantId() { - return tenantId; - } - - @Override - protected String getDispatcherName() { - return DefaultActorService.TENANT_RULE_DISPATCHER_NAME; - } - - @Override - protected FetchFunction getFetchEntitiesFunction() { - return link -> service.findTenantRuleChains(tenantId, link); - } -} diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 1ef33467b2..717dcecc19 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; @Slf4j public class StatsActor extends ContextAwareActor { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 3a719ca857..e54b955419 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -30,7 +30,6 @@ import org.thingsboard.server.actors.device.DeviceActorCreator; import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -43,6 +42,7 @@ import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; @@ -51,12 +51,12 @@ import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { - private final TenantId tenantId; private final BiMap deviceActors; + private boolean isRuleEngine; + private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { - super(systemContext, new TenantRuleChainManager(systemContext, tenantId)); - this.tenantId = tenantId; + super(systemContext, tenantId); this.deviceActors = HashBiMap.create(); } @@ -69,7 +69,11 @@ public class TenantActor extends RuleChainManagerActor { public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - initRuleChains(); + isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + if (isRuleEngine) { + initRuleChains(); + } log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); @@ -115,7 +119,6 @@ public class TenantActor extends RuleChainManagerActor { onToDeviceActorMsg((DeviceAwareMsg) msg); break; case RULE_CHAIN_TO_RULE_CHAIN_MSG: - case REMOTE_TO_RULE_CHAIN_TELL_NEXT_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; default: @@ -129,16 +132,19 @@ public class TenantActor extends RuleChainManagerActor { } private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { + if (!isRuleEngine) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } TbMsg tbMsg = msg.getTbMsg(); if (tbMsg.getRuleChainId() == null) { - if (ruleChainManager.getRootChainActor() != null) { - ruleChainManager.getRootChainActor().tell(msg, self()); + if (getRootChainActor() != null) { + getRootChainActor().tell(msg, self()); } else { - tbMsg.getCallback().onFailure(new RuntimeException("No Root Rule Chain available!")); + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!")); log.info("[{}] No Root Chain: {}", tenantId, msg); } } else { - ActorRef ruleChainActor = ruleChainManager.get(tbMsg.getRuleChainId()); + ActorRef ruleChainActor = get(tbMsg.getRuleChainId()); if (ruleChainActor != null) { ruleChainActor.tell(msg, self()); } else { @@ -150,24 +156,29 @@ public class TenantActor extends RuleChainManagerActor { } private void onRuleChainMsg(RuleChainAwareMsg msg) { - ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); + getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self()); } private void onToDeviceActorMsg(DeviceAwareMsg msg) { + if (!isCore) { + log.warn("RECEIVED INVALID MESSAGE: {}", msg); + } getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender()); } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - ActorRef target = getEntityActorRef(msg.getEntityId()); - if (target != null) { - if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { - RuleChain ruleChain = systemContext.getRuleChainService(). - findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); - ruleChainManager.visit(ruleChain, target); + if (isRuleEngine) { + ActorRef target = getEntityActorRef(msg.getEntityId()); + if (target != null) { + if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { + RuleChain ruleChain = systemContext.getRuleChainService(). + findRuleChainById(tenantId, new RuleChainId(msg.getEntityId().getId())); + visit(ruleChain, target); + } + target.tell(msg, ActorRef.noSender()); + } else { + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } - target.tell(msg, ActorRef.noSender()); - } else { - log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg); } } @@ -214,15 +225,12 @@ public class TenantActor extends RuleChainManagerActor { } } - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function() { - @Override - public SupervisorStrategy.Directive apply(Throwable t) { - log.warn("[{}] Unknown failure", tenantId, t); - if (t instanceof ActorInitializationException) { - return SupervisorStrategy.stop(); - } else { - return SupervisorStrategy.resume(); - } + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> { + log.warn("[{}] Unknown failure", tenantId, t); + if (t instanceof ActorInitializationException) { + return SupervisorStrategy.stop(); + } else { + return SupervisorStrategy.resume(); } }); diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index f7d956e146..6e33133522 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -92,10 +92,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr try { SessionMetaData sessionMd = internalSessionMap.get(session.getId()); if (sessionMd != null) { - log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); + log.trace("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload()); webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload()); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); session.close(CloseStatus.SERVER_ERROR.withReason("Session not found!")); } } catch (IOException e) { @@ -139,7 +139,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr if (sessionMd != null) { processInWebSocketService(sessionMd.sessionRef, SessionEvent.onError(tError)); } else { - log.warn("[{}] Failed to find session", session.getId()); + log.trace("[{}] Failed to find session", session.getId()); } log.trace("[{}] Session transport error", session.getId(), tError); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 434f1d7eae..401262a243 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -25,7 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -120,8 +120,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { - log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToSubscriptionMgrMsg()) { @@ -182,7 +182,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbMsgCallback callback) { + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCoreNotificationMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); @@ -200,7 +200,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService 0 ? RpcError.values()[proto.getError()] : null; FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) , proto.getResponse(), error); @@ -215,7 +215,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); + private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbQueueRuleEngineSettings ruleEngineSettings, - TbRuleEngineQueueFactory tbRuleEngineQueueFactory, + TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + this.statisticsService = statisticsService; this.ruleEngineSettings = ruleEngineSettings; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.factory = factory; @@ -92,7 +98,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< public void init() { super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) { + consumerConfigurations.putIfAbsent(configuration.getName(), configuration); consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); + consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); } } @@ -107,7 +115,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue))); + consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue))); } @Override @@ -115,7 +123,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< consumers.values().forEach(TbQueueConsumer::unsubscribe); } - private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration) { + private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { consumersExecutor.execute(() -> { while (!stopped) { try { @@ -123,7 +131,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (msgs.isEmpty()) { continue; } - TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getAckStrategy()); + TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getName(), configuration.getAckStrategy()); TbRuleEngineProcessingDecision decision = null; boolean firstAttempt = true; while (!stopped && (firstAttempt || !decision.isCommit())) { @@ -137,21 +145,21 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } ConcurrentMap> successMap = new ConcurrentHashMap<>(); ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - + ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); allMap.forEach((id, msg) -> { - log.info("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, allMap, successMap, failedMap); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = new TbMsgPackCallback<>(id, tenantId, processingTimeoutLatch, allMap, successMap, failedMap, exceptionsMap); try { - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); } else { callback.onSuccess(); } - } catch (Throwable e) { - callback.onFailure(e); + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage())); } }); @@ -159,7 +167,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { timeout = true; } - decision = strategy.analyze(new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap)); + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap, exceptionsMap); + decision = strategy.analyze(result); + if (statsEnabled) { + stats.log(result, decision.isCommit()); + } } consumer.commit(); } catch (Exception e) { @@ -193,7 +205,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } @Override - protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbMsgCallback callback) throws Exception { + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { ToRuleEngineNotificationMsg nfMsg = msg.getValue(); if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); @@ -219,18 +231,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< relationTypes = new HashSet<>(relationTypesList); } } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes); + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); actorContext.getAppActor().tell(msg, ActorRef.noSender()); - //TODO: 2.5 before release. -// if (statsEnabled) { -// stats.log(toDeviceActorMsg); -// } } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { - stats.printStats(); + long ts = System.currentTimeMillis(); + consumerStats.forEach((queue, stats) -> { + stats.printStats(); + statisticsService.reportQueueStats(ts, stats); + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java new file mode 100644 index 0000000000..0470db4cd7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeException; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class TbMsgPackCallback implements TbMsgCallback { + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap successMap; + private final ConcurrentMap failedMap; + private final UUID id; + private final TenantId tenantId; + private final ConcurrentMap firstExceptions; + + public TbMsgPackCallback(UUID id, TenantId tenantId, + CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap successMap, + ConcurrentMap failedMap, + ConcurrentMap firstExceptions) { + this.id = id; + this.tenantId = tenantId; + this.processingTimeoutLatch = processingTimeoutLatch; + this.ackMap = ackMap; + this.successMap = successMap; + this.failedMap = failedMap; + this.firstExceptions = firstExceptions; + } + + @Override + public void onSuccess() { + log.trace("[{}] ON SUCCESS", id); + T msg = ackMap.remove(id); + if (msg != null) { + successMap.put(id, msg); + } + if (msg != null && ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } + + @Override + public void onFailure(RuleEngineException e) { + log.trace("[{}] ON FAILURE", id, e); + T msg = ackMap.remove(id); + if (msg != null) { + failedMap.put(id, msg); + firstExceptions.putIfAbsent(tenantId, e); + } + if (ackMap.isEmpty()) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index 72c5c10458..ba5a883ea0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/MsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -16,30 +16,26 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @Slf4j -public class MsgPackCallback implements TbMsgCallback { +public class TbPackCallback implements TbCallback { private final CountDownLatch processingTimeoutLatch; private final ConcurrentMap ackMap; - private final ConcurrentMap successMap; private final ConcurrentMap failedMap; private final UUID id; - public MsgPackCallback(UUID id, CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap successMap, - ConcurrentMap failedMap) { + public TbPackCallback(UUID id, + CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { this.id = id; this.processingTimeoutLatch = processingTimeoutLatch; this.ackMap = ackMap; - this.successMap = successMap; this.failedMap = failedMap; } @@ -47,9 +43,6 @@ public class MsgPackCallback implements TbMsgCallback { public void onSuccess() { log.trace("[{}] ON SUCCESS", id); T msg = ackMap.remove(id); - if (msg != null) { - successMap.put(id, msg); - } if (msg != null && ackMap.isEmpty()) { processingTimeoutLatch.countDown(); } @@ -57,7 +50,7 @@ public class MsgPackCallback implements TbMsgCallback { @Override public void onFailure(Throwable t) { - log.trace("[{}] ON FAILURE", id); + log.trace("[{}] ON FAILURE", id, t); T msg = ackMap.remove(id); if (msg != null) { failedMap.put(id, msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 27792c5a61..22f4edbbba 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -15,38 +15,119 @@ */ package org.thingsboard.server.service.queue; +import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeException; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @Slf4j +@Data public class TbRuleEngineConsumerStats { - private final AtomicInteger totalCounter = new AtomicInteger(0); - private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); - private final AtomicInteger postAttributesCounter = new AtomicInteger(0); - private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); + public static final String TOTAL_MSGS = "totalMsgs"; + public static final String SUCCESSFUL_MSGS = "successfulMsgs"; + public static final String TMP_TIMEOUT = "tmpTimeout"; + public static final String TMP_FAILED = "tmpFailed"; + public static final String TIMEOUT_MSGS = "timeoutMsgs"; + public static final String FAILED_MSGS = "failedMsgs"; + public static final String SUCCESSFUL_ITERATIONS = "successfulIterations"; + public static final String FAILED_ITERATIONS = "failedIterations"; - public void log(TransportProtos.TransportToRuleEngineMsg msg) { - totalCounter.incrementAndGet(); - if (msg.hasPostTelemetry()) { - postTelemetryCounter.incrementAndGet(); - } - if (msg.hasPostAttributes()) { - postAttributesCounter.incrementAndGet(); - } - if (msg.hasToServerRPCCallRequest()) { - toServerRPCCallRequestCounter.incrementAndGet(); + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger successIterationsCounter = new AtomicInteger(0); + private final AtomicInteger failedIterationsCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + private final ConcurrentMap tenantStats = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); + + private final String queueName; + + public TbRuleEngineConsumerStats(String queueName) { + this.queueName = queueName; + counters.put(TOTAL_MSGS, totalMsgCounter); + counters.put(SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(FAILED_MSGS, failedMsgCounter); + + counters.put(TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TMP_FAILED, tmpFailedMsgCounter); + counters.put(SUCCESSFUL_ITERATIONS, successIterationsCounter); + counters.put(FAILED_ITERATIONS, failedIterationsCounter); + } + + public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { + int success = msg.getSuccessMap().size(); + int pending = msg.getPendingMap().size(); + int failed = msg.getFailureMap().size(); + totalMsgCounter.addAndGet(success + pending + failed); + successMsgCounter.addAndGet(success); + msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); + if (finalIterationForPack) { + if (pending > 0 || failed > 0) { + timeoutMsgCounter.addAndGet(pending); + failedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); + } + if (failed > 0) { + msg.getFailureMap().values().forEach(m -> getTenantStats(m).logFailed()); + } + failedIterationsCounter.incrementAndGet(); + } else { + successIterationsCounter.incrementAndGet(); + } + } else { + failedIterationsCounter.incrementAndGet(); + tmpTimeoutMsgCounter.addAndGet(pending); + tmpFailedMsgCounter.addAndGet(failed); + if (pending > 0) { + msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); + } + if (failed > 0) { + msg.getFailureMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + } } + msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); + } + + private TbTenantRuleEngineStats getTenantStats(TbProtoQueueMsg m) { + ToRuleEngineMsg reMsg = m.getValue(); + return tenantStats.computeIfAbsent(new UUID(reMsg.getTenantIdMSB(), reMsg.getTenantIdLSB()), TbTenantRuleEngineStats::new); } public void printStats() { - int total = totalCounter.getAndSet(0); + int total = totalMsgCounter.get(); if (total > 0) { - log.info("Transport total [{}] telemetry [{}] attributes [{}] toServerRpc [{}]", - total, postTelemetryCounter.getAndSet(0), - postAttributesCounter.getAndSet(0), toServerRPCCallRequestCounter.getAndSet(0)); + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("] "); + }); + log.info("[{}] Stats: {}", queueName, stats); } } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + tenantStats.clear(); + tenantExceptions.clear(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java new file mode 100644 index 0000000000..fb36f5064b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbTenantRuleEngineStats.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +@Data +public class TbTenantRuleEngineStats { + + private final UUID tenantId; + + private final AtomicInteger totalMsgCounter = new AtomicInteger(0); + private final AtomicInteger successMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpTimeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger tmpFailedMsgCounter = new AtomicInteger(0); + + private final AtomicInteger timeoutMsgCounter = new AtomicInteger(0); + private final AtomicInteger failedMsgCounter = new AtomicInteger(0); + + private final Map counters = new HashMap<>(); + + public TbTenantRuleEngineStats(UUID tenantId) { + this.tenantId = tenantId; + counters.put(TbRuleEngineConsumerStats.TOTAL_MSGS, totalMsgCounter); + counters.put(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS, successMsgCounter); + counters.put(TbRuleEngineConsumerStats.TIMEOUT_MSGS, timeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.FAILED_MSGS, failedMsgCounter); + + counters.put(TbRuleEngineConsumerStats.TMP_TIMEOUT, tmpTimeoutMsgCounter); + counters.put(TbRuleEngineConsumerStats.TMP_FAILED, tmpFailedMsgCounter); + } + + public void logSuccess() { + totalMsgCounter.incrementAndGet(); + successMsgCounter.incrementAndGet(); + } + + public void logFailed() { + totalMsgCounter.incrementAndGet(); + failedMsgCounter.incrementAndGet(); + } + + public void logTimeout() { + totalMsgCounter.incrementAndGet(); + timeoutMsgCounter.incrementAndGet(); + } + + public void logTmpFailed() { + totalMsgCounter.incrementAndGet(); + tmpFailedMsgCounter.incrementAndGet(); + } + + public void logTmpTimeout() { + totalMsgCounter.incrementAndGet(); + tmpTimeoutMsgCounter.incrementAndGet(); + } + + public void printStats() { + int total = totalMsgCounter.get(); + if (total > 0) { + StringBuilder stats = new StringBuilder(); + counters.forEach((label, value) -> { + stats.append(label).append(" = [").append(value.get()).append("]"); + }); + log.info("[{}] Stats: {}", tenantId, stats); + } + } + + public void reset() { + counters.values().forEach(counter -> counter.set(0)); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 0df0517232..a238e34337 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -22,12 +22,12 @@ import org.springframework.context.event.EventListener; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; -import org.thingsboard.server.service.queue.MsgPackCallback; +import org.thingsboard.server.service.queue.TbPackCallback; import javax.annotation.PreDestroy; import java.util.List; @@ -95,8 +95,8 @@ public abstract class AbstractConsumerService> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); pendingMap.forEach((id, msg) -> { - log.info("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbMsgCallback callback = new MsgPackCallback<>(id, processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>(), failedMap); + log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); try { handleNotification(id, msg, callback); } catch (Throwable e) { @@ -124,7 +124,7 @@ public abstract class AbstractConsumerService msg, TbMsgCallback callback) throws Exception; + protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; @PreDestroy public void destroy() { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index c17cf54bca..e67936e294 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.queue.processing; import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -25,24 +27,28 @@ import java.util.concurrent.ConcurrentMap; public class TbRuleEngineProcessingResult { @Getter - private boolean success; + private final boolean success; @Getter - private boolean timeout; + private final boolean timeout; @Getter - private ConcurrentMap> pendingMap; + private final ConcurrentMap> pendingMap; @Getter - private ConcurrentMap> successMap; + private final ConcurrentMap> successMap; @Getter - private ConcurrentMap> failureMap; + private final ConcurrentMap> failureMap; + @Getter + private final ConcurrentMap exceptionsMap; public TbRuleEngineProcessingResult(boolean timeout, ConcurrentMap> pendingMap, ConcurrentMap> successMap, - ConcurrentMap> failureMap) { + ConcurrentMap> failureMap, + ConcurrentMap exceptionsMap) { this.timeout = timeout; this.pendingMap = pendingMap; this.successMap = successMap; this.failureMap = failureMap; + this.exceptionsMap = exceptionsMap; this.success = !timeout && pendingMap.isEmpty() && failureMap.isEmpty(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 85a44794d8..1096bfbf7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -32,24 +32,25 @@ import java.util.concurrent.TimeUnit; @Slf4j public class TbRuleEngineProcessingStrategyFactory { - public TbRuleEngineProcessingStrategy newInstance(TbRuleEngineQueueAckStrategyConfiguration configuration) { + public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { switch (configuration.getType()) { case "SKIP_ALL": - return new SkipStrategy(); + return new SkipStrategy(name); case "RETRY_ALL": - return new RetryStrategy(true, true, true, configuration); + return new RetryStrategy(name, true, true, true, configuration); case "RETRY_FAILED": - return new RetryStrategy(false, true, false, configuration); + return new RetryStrategy(name, false, true, false, configuration); case "RETRY_TIMED_OUT": - return new RetryStrategy(false, false, true, configuration); + return new RetryStrategy(name, false, false, true, configuration); case "RETRY_FAILED_AND_TIMED_OUT": - return new RetryStrategy(false, true, true, configuration); + return new RetryStrategy(name, false, true, true, configuration); default: throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); } } private static class RetryStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; private final boolean retrySuccessful; private final boolean retryFailed; private final boolean retryTimeout; @@ -60,7 +61,8 @@ public class TbRuleEngineProcessingStrategyFactory { private int initialTotalCount; private int retryCount; - public RetryStrategy(boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) { + this.queueName = queueName; this.retrySuccessful = retrySuccessful; this.retryFailed = retryFailed; this.retryTimeout = retryTimeout; @@ -80,10 +82,10 @@ public class TbRuleEngineProcessingStrategyFactory { retryCount++; double failedCount = result.getFailureMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { - log.info("Skip reprocess of the rule engine pack due to max retries"); + log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("Skip reprocess of the rule engine pack due to max allowed failure percentage"); + log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); return new TbRuleEngineProcessingDecision(true, null); } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); @@ -96,8 +98,7 @@ public class TbRuleEngineProcessingStrategyFactory { if (retrySuccessful) { result.getSuccessMap().forEach(toReprocess::put); } - log.info("Going to reprocess {} messages", toReprocess.size()); - //TODO: 2.5 Log most popular rule nodes by error count; + log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); } @@ -116,9 +117,15 @@ public class TbRuleEngineProcessingStrategyFactory { private static class SkipStrategy implements TbRuleEngineProcessingStrategy { + private final String queueName; + + public SkipStrategy(String name) { + this.queueName = name; + } + @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("Skip reprocess of the rule engine pack"); + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailureMap().size(), result.getPendingMap().size()); return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java index 78ea0c5660..c1e5e2e038 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/FromDeviceRpcResponse.java @@ -19,7 +19,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import org.thingsboard.rule.engine.api.RpcError; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java index 3a1ea599e6..3cfe9bc1f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/ToDeviceRpcRequestActorMsg.java @@ -22,11 +22,8 @@ import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import java.util.Optional; - /** * Created by ashvayka on 16.04.18. */ diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 32980c0064..b4c5719844 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -55,7 +55,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -242,7 +242,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbMsgCallback callback) { + public void onQueueMsg(TransportProtos.DeviceStateServiceMsgProto proto, TbCallback callback) { try { TenantId tenantId = new TenantId(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); DeviceId deviceId = new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index c7aadda963..16c699b38a 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; /** * Created by ashvayka on 01.05.18. @@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener CALLBACK = new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to persist statistics", t); + } + }; + + private final TbServiceInfoProvider serviceInfoProvider; + private final TelemetrySubscriptionService tsService; + private final Lock lock = new ReentrantLock(); + private final AssetService assetService; + private final ConcurrentMap tenantQueueAssets; + + public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { + this.tsService = tsService; + this.serviceInfoProvider = serviceInfoProvider; + this.assetService = assetService; + this.tenantQueueAssets = new ConcurrentHashMap<>(); + } + + @Override + public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { + String queueName = ruleEngineStats.getQueueName(); + ruleEngineStats.getTenantStats().forEach((id, stats) -> { + TenantId tenantId = new TenantId(id); + AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + if (stats.getTotalMsgCounter().get() > 0) { + List tsList = stats.getCounters().entrySet().stream() + .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) + .collect(Collectors.toList()); + if (!tsList.isEmpty()) { + tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + } + } + }); + ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { + TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + }); + ruleEngineStats.reset(); + } + + private AssetId getServiceAssetId(TenantId tenantId, String queueName) { + TenantQueueKey key = new TenantQueueKey(tenantId, queueName); + AssetId assetId = tenantQueueAssets.get(key); + if (assetId == null) { + lock.lock(); + try { + assetId = tenantQueueAssets.get(key); + if (assetId == null) { + Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); + if (asset == null) { + asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); + asset.setType("TbServiceQueue"); + asset = assetService.saveAsset(asset); + } + assetId = asset.getId(); + tenantQueueAssets.put(key, assetId); + } + } finally { + lock.unlock(); + } + } + return assetId; + } + + @Data + private static class TenantQueueKey { + private final TenantId tenantId; + private final String queueName; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java similarity index 72% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java rename to application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java index 4b00284c34..88573844c9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerType.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/RuleEngineStatisticsService.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.server.service.stats; -/** - * Created by ashvayka on 23.09.18. - */ -public enum ServerType { - CORE +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; + +public interface RuleEngineStatisticsService { + + void reportQueueStats(long ts, TbRuleEngineConsumerStats stats); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index b6955e40a9..0d151c3f46 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -123,7 +123,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void addSubscription(TbSubscription subscription, TbMsgCallback callback) { + public void addSubscription(TbSubscription subscription, TbCallback callback) { log.trace("[{}][{}][{}] Registering remote subscription for entity [{}]", subscription.getServiceId(), subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); @@ -151,7 +151,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback) { + public void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback) { log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId); Map sessionSubscriptions = subscriptionsByWsSessionId.get(sessionId); if (sessionSubscriptions != null) { @@ -189,7 +189,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback) { + public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { onLocalSubUpdate(entityId, s -> { if (TbSubscriptionType.TIMESERIES.equals(s.getType())) { @@ -213,7 +213,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback) { + public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { onLocalSubUpdate(entityId, s -> { if (TbSubscriptionType.ATTRIBUTES.equals(s.getType())) { @@ -261,7 +261,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer if (subscriptionUpdate != null && !subscriptionUpdate.isEmpty()) { if (serviceId.equals(s.getServiceId())) { SubscriptionUpdate update = new SubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate); - localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbMsgCallback.EMPTY); + localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY); } else { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId()); toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate), null); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index e4d0d838ca..12c5e206e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; @@ -135,7 +135,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (currentPartitions.contains(tpi)) { // Subscription is managed on the same server; if (pushToLocalService) { - subscriptionManagerService.addSubscription(subscription, TbMsgCallback.EMPTY); + subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY); } } else { // Push to the queue; @@ -145,7 +145,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } @Override - public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback) { + public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) { TbSubscription subscription = subscriptionsBySessionId .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId()); if (subscription != null) { @@ -177,7 +177,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId()); if (currentPartitions.contains(tpi)) { // Subscription is managed on the same server; - subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbMsgCallback.EMPTY); + subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY); } else { // Push to the queue; TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index aa1824c855..e20b707348 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -21,18 +21,18 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.List; public interface SubscriptionManagerService extends ApplicationListener { - void addSubscription(TbSubscription subscription, TbMsgCallback callback); + void addSubscription(TbSubscription subscription, TbCallback callback); - void cancelSubscription(String sessionId, int subscriptionId, TbMsgCallback callback); + void cancelSubscription(String sessionId, int subscriptionId, TbCallback callback); - void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbMsgCallback callback); + void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback); - void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbMsgCallback callback); + void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java index b6bee14a32..58fba24250 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -17,7 +17,7 @@ package org.thingsboard.server.service.subscription; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; public interface TbLocalSubscriptionService { @@ -28,7 +28,7 @@ public interface TbLocalSubscriptionService { void cancelAllSessionSubscriptions(String sessionId); - void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbMsgCallback callback); + void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback); void onApplicationEvent(PartitionChangeEvent event); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 77b0967966..b3c728a25b 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -41,7 +41,7 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -166,7 +166,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbMsgCallback.EMPTY); + subscriptionManagerService.get().onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY); } else { log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } @@ -180,7 +180,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); if (currentPartitions.contains(tpi)) { if (subscriptionManagerService.isPresent()) { - subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbMsgCallback.EMPTY); + subscriptionManagerService.get().onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); } else { log.warn("Possible misconfiguration because subscriptionManagerService is null!"); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index e905559516..d652a3f1c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -17,11 +17,7 @@ package org.thingsboard.server.service.telemetry; import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; -import org.thingsboard.server.service.telemetry.sub.SubscriptionState; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java b/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java deleted file mode 100644 index 7f7db9430d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/sub/Subscription.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.telemetry.sub; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.cluster.ServerAddress; -import org.thingsboard.server.service.telemetry.TelemetryFeature; - -import java.util.Map; - -@Data -@AllArgsConstructor -public class Subscription { - - private final SubscriptionState sub; - private final boolean local; - private ServerAddress server; - private long startTime; - private long endTime; - - public Subscription(SubscriptionState sub, boolean local, ServerAddress server) { - this(sub, local, server, 0L, 0L); - } - - public String getWsSessionId() { - return getSub().getWsSessionId(); - } - - public int getSubscriptionId() { - return getSub().getSubscriptionId(); - } - - public EntityId getEntityId() { - return getSub().getEntityId(); - } - - public TelemetryFeature getType() { - return getSub().getType(); - } - - public String getScope() { - return getSub().getScope(); - } - - public boolean isAllKeys() { - return getSub().isAllKeys(); - } - - public Map getKeyStates() { - return getSub().getKeyStates(); - } - - public void setKeyState(String key, long ts) { - getSub().getKeyStates().put(key, ts); - } - - @Override - public String toString() { - return "Subscription{" + - "sub=" + sub + - ", local=" + local + - ", server=" + server + - '}'; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java index 783dd5181a..477f115a96 100644 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java @@ -22,7 +22,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -114,13 +113,6 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ // } } - @Override - public void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] data) { - endLocalTransaction(TbMsg.fromBytes(data, null), msg -> { - }, error -> { - }); - } - private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { queue.offer(transactionTask); timeoutQueue.offer(transactionTask); @@ -230,9 +222,4 @@ public class BaseRuleChainTransactionService implements RuleChainTransactionServ callbackExecutor.executeAsync(task); } - private void sendTransactionEventToRemoteServer(TbMsg msg, ServerAddress address) { - log.trace("[{}][{}] Originator is monitored on other server: {}", msg.getTransactionData().getOriginatorId(), msg.getTransactionData().getTransactionId(), address); - //TODO 2.5 -// clusterRpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TRANSACTION_SERVICE_MESSAGE, TbMsg.toByteArray(msg)); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java index 16e0dd67f2..ba4e7baab1 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TbCallback; import java.io.Serializable; import java.util.UUID; @@ -37,9 +37,9 @@ public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAware private final TenantId tenantId; private final DeviceId deviceId; private final TransportToDeviceActorMsg msg; - private final TbMsgCallback callback; + private final TbCallback callback; - public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbMsgCallback callback) { + public TransportToDeviceActorMsgWrapper(TransportToDeviceActorMsg msg, TbCallback callback) { this.msg = msg; this.callback = callback; this.tenantId = new TenantId(new UUID(msg.getSessionInfo().getTenantIdMSB(), msg.getSessionInfo().getTenantIdLSB())); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 3d47db432f..8389fa5d71 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,8 +27,8 @@ - - + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 04c9205a12..79efacba97 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -561,7 +561,7 @@ queue: poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: # TODO 2.5: specify correct ENV variable names. - @@ -577,7 +577,7 @@ queue: failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "HighPriority" + name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" @@ -594,7 +594,7 @@ queue: poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: - type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine or tb-transport + type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 5ecb5fb415..7fe63827eb 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -35,8 +35,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index bc4db3b535..78129917ef 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index dec6f6b50b..47e09b812f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -33,11 +33,6 @@ public enum MsgType { APP_INIT_MSG, - /** - * All messages, could be send to cluster - */ - SEND_TO_CLUSTER_MSG, - /** * ADDED/UPDATED/DELETED events for main entities. * diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 7230804773..84110ed163 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,7 +17,6 @@ package org.thingsboard.server.common.msg; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -29,7 +28,6 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.nio.ByteBuffer; import java.util.UUID; /** diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java deleted file mode 100644 index 80674f9bce..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/ServerAddress.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.cluster; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - * @author Andrew Shvayka - */ -@Data -@EqualsAndHashCode -public class ServerAddress implements Comparable, Serializable { - - private final String host; - private final int port; - private final ServerType serverType; - - @Override - public int compareTo(ServerAddress o) { - int result = this.host.compareTo(o.host); - if (result == 0) { - result = this.port - o.port; - } - return result; - } - - @Override - public String toString() { - return '[' + host + ':' + port + ']'; - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java index 49b30a5992..fca1c51995 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/QueueToRuleEngineMsg.java @@ -33,6 +33,7 @@ public final class QueueToRuleEngineMsg implements TbActorMsg { private final TenantId tenantId; private final TbMsg tbMsg; private final Set relationTypes; + private final String failureMessage; @Override public MsgType getMsgType() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java new file mode 100644 index 0000000000..dd720251bb --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RuleEngineException extends Exception { + protected static final ObjectMapper mapper = new ObjectMapper(); + + public RuleEngineException(String message) { + super(message != null ? message : "Unknown"); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java new file mode 100644 index 0000000000..3f437dd1d7 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.rule.RuleNode; + +@Slf4j +public class RuleNodeException extends RuleEngineException { + @Getter + private final String ruleChainName; + @Getter + private final String ruleNodeName; + @Getter + private final RuleChainId ruleChainId; + @Getter + private final RuleNodeId ruleNodeId; + + public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { + super(message); + this.ruleChainName = ruleChainName; + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } + + public String toJsonString() { + try { + return mapper.writeValueAsString(mapper.createObjectNode() + .put("ruleNodeId", ruleNodeId.toString()) + .put("ruleChainId", ruleChainId.toString()) + .put("ruleNodeName", ruleNodeName) + .put("ruleChainName", ruleChainName) + .put("message", getMessage())); + } catch (JsonProcessingException e) { + log.warn("Failed to serialize exception ", e); + throw new RuntimeException(e); + } + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java new file mode 100644 index 0000000000..4c25f27a3e --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.queue; + +public interface TbCallback { + + TbCallback EMPTY = new TbCallback() { + + @Override + public void onSuccess() { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + + void onSuccess(); + + void onFailure(Throwable t); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java index 4f0e383fe9..9e8d5ae6b8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbMsgCallback.java @@ -25,13 +25,13 @@ public interface TbMsgCallback { } @Override - public void onFailure(Throwable t) { + public void onFailure(RuleEngineException e) { } }; void onSuccess(); - void onFailure(Throwable t); + void onFailure(RuleEngineException e); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java index 552eca84a0..f860d8dcc9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueTbMsgCallbackWrapper.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.common; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; @@ -40,6 +42,6 @@ public class MultipleTbQueueTbMsgCallbackWrapper implements TbQueueCallback { @Override public void onFailure(Throwable t) { - tbMsgCallback.onFailure(t); + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java index c372f64b3d..8bd9c3516b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbQueueTbMsgCallbackWrapper.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.queue.common; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import java.util.concurrent.atomic.AtomicInteger; - public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { private final TbMsgCallback tbMsgCallback; @@ -36,6 +36,6 @@ public class TbQueueTbMsgCallbackWrapper implements TbQueueCallback { @Override public void onFailure(Throwable t) { - tbMsgCallback.onFailure(t); + tbMsgCallback.onFailure(new RuleEngineException(t.getMessage())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 1337a1cc74..3558374af5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -86,24 +86,6 @@ public class ConsistentHashPartitionService implements PartitionService { partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); } -// public Set getCurrentPartitions(ServiceType serviceType) { -// ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); -// TenantId tenantId = getSystemOrIsolatedTenantId(currentService); -// ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceType, tenantId); -// List partitions = myPartitions.get(serviceQueueKey); -// Set topicPartitions = new LinkedHashSet<>(); -// for (Integer partition : partitions) { -// TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder(); -// tpi.topic(partitionTopics.get(serviceType)); -// tpi.partition(partition); -// if (!tenantId.isNullUid()) { -// tpi.tenantId(tenantId); -// } -// topicPartitions.add(tpi.build()); -// } -// return topicPartitions; -// } - @Override public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { return resolve(new ServiceQueue(serviceType), tenantId, entityId); @@ -131,15 +113,7 @@ public class ConsistentHashPartitionService implements PartitionService { Map> circles = new HashMap<>(); addNode(circles, currentService); for (ServiceInfo other : otherServices) { - TenantId tenantId = getSystemOrIsolatedTenantId(other); addNode(circles, other); - if (!tenantId.isNullUid()) { - isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); - for (String serviceType : other.getServiceTypesList()) { - isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); - } - - } } ConcurrentMap> oldPartitions = myPartitions; TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); @@ -214,6 +188,11 @@ public class ConsistentHashPartitionService implements PartitionService { } } + @Override + public Set getIsolatedTenants(ServiceType serviceType) { + throw new RuntimeException("Not Implemented!"); + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { @@ -280,6 +259,12 @@ public class ConsistentHashPartitionService implements PartitionService { private void addNode(Map> circles, ServiceInfo instance) { TenantId tenantId = getSystemOrIsolatedTenantId(instance); + if (!tenantId.isNullUid()) { + isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); + for (String serviceType : instance.getServiceTypesList()) { + isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); + } + } for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 15cf33bb82..1c60a03cdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -34,6 +34,7 @@ import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -58,6 +59,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { private List serviceTypes; private ServiceInfo serviceInfo; + private TenantId isolatedTenant; @PostConstruct public void init() { @@ -80,6 +82,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { UUID tenantId; if (!StringUtils.isEmpty(tenantIdStr)) { tenantId = UUID.fromString(tenantIdStr); + isolatedTenant = new TenantId(tenantId); } else { tenantId = TenantId.NULL_UUID; } @@ -103,4 +106,14 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { public ServiceInfo getServiceInfo() { return serviceInfo; } + + @Override + public boolean isService(ServiceType serviceType) { + return serviceTypes.contains(serviceType); + } + + @Override + public Optional getIsolatedTenant() { + return Optional.ofNullable(isolatedTenant); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index e3b3e76b80..7c6900d8a8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -47,6 +47,8 @@ public interface PartitionService { */ Set getAllServiceIds(ServiceType serviceType); + Set getIsolatedTenants(ServiceType serviceType); + /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. * This topic is likely to have single partition, and is always assigned to the service. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 053dd644c5..fcc7c7a60d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -15,12 +15,20 @@ */ package org.thingsboard.server.queue.discovery; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import java.util.Optional; + public interface TbServiceInfoProvider { String getServiceId(); ServiceInfo getServiceInfo(); + boolean isService(ServiceType serviceType); + + Optional getIsolatedTenant(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index 48eeb5ba2a..95540798fc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,7 +17,9 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @@ -26,6 +28,7 @@ import java.util.List; @Slf4j @Data +@EnableAutoConfiguration @Configuration @ConfigurationProperties(prefix = "queue.rule-engine") public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 55d376b63b..e36c4e36a9 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -388,6 +388,7 @@ message ToRuleEngineMsg { int64 tenantIdLSB = 2; bytes tbMsg = 3; repeated string relationTypes = 4; + string failureMessage = 5; } message ToRuleEngineNotificationMsg { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java index bb0210b1ad..1bb56adb1d 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java @@ -16,7 +16,6 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.cluster.ServerAddress; import java.util.function.Consumer; @@ -26,6 +25,4 @@ public interface RuleChainTransactionService { void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - void onRemoteTransactionMsg(ServerAddress serverAddress, byte[] bytes); - } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 72a552cb4f..4299aeb867 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -26,10 +26,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -116,6 +114,8 @@ public interface TbContext { */ void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + void enqueueForTellFailure(TbMsg msg, String failureMessage); + void enqueueForTellNext(TbMsg msg, String relationType); void enqueueForTellNext(TbMsg msg, Set relationTypes); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java new file mode 100644 index 0000000000..9351e970ed --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "acknowledge", + configClazz = EmptyNodeConfiguration.class, + nodeDescription = "Acknowledges the incoming message", + nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.") + +public class TbAckNode implements TbNode { + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.ack(msg); + ctx.tellSuccess(msg); + } + + @Override + public void destroy() { + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java new file mode 100644 index 0000000000..0364251076 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.flow; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.ScriptEngine; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TbRelationTypes; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +import static org.thingsboard.common.util.DonAsynchron.withCallback; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "checkpoint", + configClazz = TbCheckpointNodeConfiguration.class, + nodeDescription = "transfers the message to another queue", + nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.") + +public class TbCheckpointNode implements TbNode { + + private TbCheckpointNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + ctx.enqueueForTellNext(msg, config.getQueueName(), TbRelationTypes.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); + } + + @Override + public void destroy() { + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java similarity index 53% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java rename to rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java index 3326c33aa3..eb5b8e29d9 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cluster/SendToClusterMsg.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java @@ -13,28 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.cluster; +package org.thingsboard.rule.engine.flow; import lombok.Data; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.rule.engine.api.NodeConfiguration; @Data -public class SendToClusterMsg implements TbActorMsg { - - private TbActorMsg msg; - private EntityId entityId; - - public SendToClusterMsg(EntityId entityId, TbActorMsg msg) { - this.entityId = entityId; - this.msg = msg; - } +public class TbCheckpointNodeConfiguration implements NodeConfiguration { + private String queueName; @Override - public MsgType getMsgType() { - return MsgType.SEND_TO_CLUSTER_MSG; + public TbCheckpointNodeConfiguration defaultConfiguration() { + TbCheckpointNodeConfiguration configuration = new TbCheckpointNodeConfiguration(); + configuration.setQueueName("HighPriority"); + return configuration; } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java index 88b060fe5c..ad1d399e76 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCRequestNode.java @@ -116,8 +116,7 @@ public class TbSendRPCRequestNode implements TbNode { ctx.enqueueForTellNext(next, TbRelationTypes.SUCCESS); } else { TbMsg next = ctx.newMsg(msg.getType(), msg.getOriginator(), msg.getMetaData(), wrap("error", ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.tellFailure(next, new RuntimeException(ruleEngineDeviceRpcResponse.getError().get().name())); - ctx.enqueueForTellNext(next, TbRelationTypes.FAILURE); + ctx.enqueueForTellFailure(next, ruleEngineDeviceRpcResponse.getError().get().name()); } }); ctx.ack(msg); diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index a875d56782..573fe6d5df 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -102,20 +102,44 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" - rule_engine: + rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" - poll_interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: # TODO 2.5: specify correct ENV variable names. + - + name: "Main" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - + name: "HighPriority" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + ack-strategy: + type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; transport: notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: - type: "${TB_SERVICE_TYPE:tb-transport}" # monolith or tb-core or tb-rule-engine or tb-transport + type: "${TB_SERVICE_TYPE:tb-transport}" # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file From 5bebf098ab0dbc3b7ef5cebeb6db549cde3cc900 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 9 Apr 2020 13:03:30 +0300 Subject: [PATCH 046/196] Ability to reset blacklisted functions --- .../script/AbstractJsInvokeService.java | 48 +++++++++++++++---- .../AbstractNashornJsInvokeService.java | 4 +- .../script/NashornJsInvokeService.java | 10 ++++ .../service/script/RemoteJsInvokeService.java | 11 +++++ .../src/main/resources/thingsboard.yml | 12 +++-- .../script/TestNashornJsInvokeService.java | 5 ++ 6 files changed, 73 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index 31827f542e..daa74d013f 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractJsInvokeService implements JsInvokeService { protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); - protected Map blackListedFunctions = new ConcurrentHashMap<>(); + protected Map blackListedFunctions = new ConcurrentHashMap<>(); @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { @@ -78,25 +78,53 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { protected abstract int getMaxErrors(); + protected abstract long getMaxBlacklistDuration(); + protected void onScriptExecutionError(UUID scriptId) { - blackListedFunctions.computeIfAbsent(scriptId, key -> new AtomicInteger(0)).incrementAndGet(); + blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); } private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { - switch (scriptType) { - case RULE_NODE_SCRIPT: - return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); - default: - throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); + if (scriptType == JsScriptType.RULE_NODE_SCRIPT) { + return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames); } + throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); } private boolean isBlackListed(UUID scriptId) { - if (blackListedFunctions.containsKey(scriptId)) { - AtomicInteger errorCount = blackListedFunctions.get(scriptId); - return errorCount.get() >= getMaxErrors(); + BlackListInfo errorCount = blackListedFunctions.get(scriptId); + if (errorCount != null) { + if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { + blackListedFunctions.remove(scriptId); + return false; + } else { + return errorCount.get() >= getMaxErrors(); + } } else { return false; } } + + private class BlackListInfo { + private final AtomicInteger counter; + private long expirationTime; + + private BlackListInfo() { + this.counter = new AtomicInteger(0); + } + + public int get() { + return counter.get(); + } + + public int incrementAndGet() { + int result = counter.incrementAndGet(); + expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration(); + return result; + } + + public long getExpirationTime() { + return expirationTime; + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index 49a6304ebb..a9acfa4f42 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -55,8 +55,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private final AtomicInteger jsEvalMsgs = new AtomicInteger(0); private final AtomicInteger jsFailedMsgs = new AtomicInteger(0); private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0); - private final FutureCallback evalCallback = new JsStatCallback(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); - private final FutureCallback invokeCallback = new JsStatCallback(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); + private final FutureCallback invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java index 909565609c..66a14cc827 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/NashornJsInvokeService.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import java.util.concurrent.TimeUnit; + @Slf4j @ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true) @Service @@ -37,6 +39,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @Value("${js.local.max_errors}") private int maxErrors; + @Value("${js.local.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Override protected boolean useJsSandbox() { return useJsSandbox; @@ -56,4 +61,9 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 1b5332bcfc..b0554decea 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -36,6 +36,7 @@ import javax.annotation.PreDestroy; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -66,6 +67,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Value("${js.remote.max_errors}") private int maxErrors; + @Value("${js.remote.max_black_list_duration_sec:60}") + private int maxBlackListDurationSec; + @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; @@ -205,6 +209,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @Override public void onFailure(Throwable t) { + onScriptExecutionError(scriptId); if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { kafkaTimeoutMsgs.incrementAndGet(); } @@ -216,6 +221,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { if (invokeResult.getSuccess()) { return invokeResult.getResult(); } else { + onScriptExecutionError(scriptId); log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails()); throw new RuntimeException(invokeResult.getErrorDetails()); } @@ -245,4 +251,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } + @Override + protected long getMaxBlacklistDuration() { + return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec); + } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e4a234fdb9..f08d2baf46 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -425,11 +425,13 @@ js: # Specify thread pool size for JavaScript sandbox resource monitor monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}" # Maximum CPU time in milliseconds allowed for script execution - max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:3000}" + max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}" # JS Eval max request timeout. 0 - no timeout max_requests_timeout: "${LOCAL_JS_MAX_REQUEST_TIMEOUT:0}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" @@ -449,6 +451,8 @@ js: response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" + # Maximum time in seconds for black listed function to stay in the list. + max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" @@ -573,8 +577,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: # TODO 2.5: specify correct ENV variable names. - - - name: "Main" + - name: "Main" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" @@ -585,8 +588,7 @@ queue: retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - - name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" + - name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" diff --git a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java index 1065fbf6d3..8a89de5dff 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TestNashornJsInvokeService.java @@ -49,4 +49,9 @@ public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService { protected int getMaxErrors() { return maxErrors; } + + @Override + protected long getMaxBlacklistDuration() { + return 100000; + } } From ec4e2c036f8e43cb1fdf7ee96f1f044f64a99598 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 9 Apr 2020 13:09:54 +0300 Subject: [PATCH 047/196] No more failures on missing RE queue --- .../queue/discovery/ConsistentHashPartitionService.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index d4e4d87f69..0552aa05c6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -99,7 +99,14 @@ public class ConsistentHashPartitionService implements PartitionService { int hash = hashFunction.newHasher() .putLong(entityId.getId().getMostSignificantBits()) .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - int partition = Math.abs(hash % partitionSizes.get(serviceQueue)); + Integer partitionSize = partitionSizes.get(serviceQueue); + int partition; + if (partitionSize != null) { + partition = Math.abs(hash % partitionSize); + } else { + //TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes. + partition = 0; + } boolean isolatedTenant = isIsolated(serviceQueue, tenantId); TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition); return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition)); From ca193239bab7ba4331a61bd9e061f0f8998bc42c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 9 Apr 2020 18:33:44 +0300 Subject: [PATCH 048/196] RE Submit Strategies --- .../server/actors/ActorSystemContext.java | 7 +- .../queue/DefaultTbCoreConsumerService.java | 6 +- .../DefaultTbRuleEngineConsumerService.java | 77 ++++++++----- .../queue/ProcessingAttemptContext.java | 89 +++++++++++++++ .../service/queue/TbMsgPackCallback.java | 38 +----- .../queue/TbRuleEngineConsumerStats.java | 7 +- .../AbstractTbRuleEngineSubmitStrategy.java | 71 ++++++++++++ .../BatchTbRuleEngineSubmitStrategy.java | 86 ++++++++++++++ .../BurstTbRuleEngineSubmitStrategy.java | 50 ++++++++ .../service/queue/processing/IdMsgPair.java | 31 +++++ ...lByEntityIdTbRuleEngineSubmitStrategy.java | 108 ++++++++++++++++++ ...riginatorIdTbRuleEngineSubmitStrategy.java | 44 +++++++ ...lByTenantIdTbRuleEngineSubmitStrategy.java | 35 ++++++ .../SequentialTbRuleEngineSubmitStrategy.java | 73 ++++++++++++ .../TbRuleEngineProcessingResult.java | 38 +++--- ...TbRuleEngineProcessingStrategyFactory.java | 10 +- .../TbRuleEngineSubmitStrategy.java | 39 +++++++ .../TbRuleEngineSubmitStrategyFactory.java | 43 +++++++ .../rpc/DefaultTbCoreDeviceRpcService.java | 2 +- .../src/main/resources/thingsboard.yml | 46 +++++--- ...leEngineQueueAckStrategyConfiguration.java | 4 - .../TbRuleEngineQueueConfiguration.java | 3 +- ...ngineQueueSubmitStrategyConfiguration.java | 26 +++++ 23 files changed, 814 insertions(+), 119 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c266407707..d31cb62b9b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -334,7 +335,6 @@ public class ActorSystemContext { @Setter private ActorSystem actorSystem; - @Getter @Setter private ActorRef appActor; @@ -361,6 +361,8 @@ public class ActorSystemContext { config = ConfigFactory.parseResources(AKKA_CONF_FILE_NAME).withFallback(ConfigFactory.load()); } + + public Scheduler getScheduler() { return actorSystem.scheduler(); } @@ -535,4 +537,7 @@ public class ActorSystemContext { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } + public void tell(TbActorMsg tbActorMsg, ActorRef sender) { + appActor.tell(tbActorMsg, sender); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 401262a243..263e6eab24 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -137,7 +137,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } @@ -194,7 +194,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } @@ -259,7 +259,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); + private ExecutorService submitExecutor; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory factory, TbQueueRuleEngineSettings ruleEngineSettings, + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, + TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + TbQueueRuleEngineSettings ruleEngineSettings, TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService) { super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); this.statisticsService = statisticsService; this.ruleEngineSettings = ruleEngineSettings; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.factory = factory; + this.submitStrategyFactory = submitStrategyFactory; + this.processingStrategyFactory = processingStrategyFactory; } @PostConstruct @@ -102,6 +111,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName())); } + submitExecutor = Executors.newSingleThreadExecutor(); + } + + @PreDestroy + public void stop() { + if (submitExecutor != null) { + submitExecutor.shutdownNow(); + } } @Override @@ -131,27 +148,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (msgs.isEmpty()) { continue; } - TbRuleEngineProcessingStrategy strategy = factory.newInstance(configuration.getName(), configuration.getAckStrategy()); - TbRuleEngineProcessingDecision decision = null; - boolean firstAttempt = true; - while (!stopped && (firstAttempt || !decision.isCommit())) { - ConcurrentMap> allMap; - if (firstAttempt) { - allMap = msgs.stream().collect( - Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - firstAttempt = false; - } else { - allMap = decision.getReprocessMap(); - } - ConcurrentMap> successMap = new ConcurrentHashMap<>(); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); - ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - CountDownLatch processingTimeoutLatch = new CountDownLatch(1); - allMap.forEach((id, msg) -> { - log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbRuleEngineSubmitStrategy submitStrategy = submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); + TbRuleEngineProcessingStrategy ackStrategy = processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); + + submitStrategy.init(msgs); + + while (!stopped) { + ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); + submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { + log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = new TbMsgPackCallback<>(id, tenantId, processingTimeoutLatch, allMap, successMap, failedMap, exceptionsMap); + TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx); try { if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { forwardToRuleEngineActor(tenantId, toRuleEngineMsg, callback); @@ -161,17 +169,24 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } catch (Exception e) { callback.onFailure(new RuleEngineException(e.getMessage())); } - }); + })); boolean timeout = false; - if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { timeout = true; } - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, allMap, successMap, failedMap, exceptionsMap); - decision = strategy.analyze(result); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(timeout, ctx); + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); if (statsEnabled) { stats.log(result, decision.isCommit()); } + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } } consumer.commit(); } catch (Exception e) { @@ -211,7 +226,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< Optional actorMsg = encodingService.decode(nfMsg.getComponentLifecycleMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); - actorContext.getAppActor().tell(actorMsg.get(), ActorRef.noSender()); + actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } else { @@ -232,7 +247,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.getAppActor().tell(msg, ActorRef.noSender()); + actorContext.tell(msg, ActorRef.noSender()); } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java new file mode 100644 index 0000000000..2073ff8c21 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ProcessingAttemptContext { + + private final TbRuleEngineSubmitStrategy submitStrategy; + + private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + @Getter + private final ConcurrentMap> pendingMap; + @Getter + private final ConcurrentMap> successMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap> failedMap = new ConcurrentHashMap<>(); + @Getter + private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); + + public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { + this.submitStrategy = submitStrategy; + this.pendingMap = submitStrategy.getPendingMap(); + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + TbProtoQueueMsg msg; + boolean empty = false; + synchronized (pendingMap) { + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingMap.isEmpty(); + } + } + if (msg != null) { + successMap.put(id, msg); + } + submitStrategy.onSuccess(id); + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { + TbProtoQueueMsg msg; + boolean empty = false; + synchronized (pendingMap) { + msg = pendingMap.remove(id); + if (msg != null) { + empty = pendingMap.isEmpty(); + } + } + if (msg != null) { + failedMap.put(id, msg); + exceptionsMap.putIfAbsent(tenantId, e); + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java index 0470db4cd7..2a6b6a658d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -27,52 +27,26 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @Slf4j -public class TbMsgPackCallback implements TbMsgCallback { - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap successMap; - private final ConcurrentMap failedMap; +public class TbMsgPackCallback implements TbMsgCallback { private final UUID id; private final TenantId tenantId; - private final ConcurrentMap firstExceptions; + private final ProcessingAttemptContext ctx; - public TbMsgPackCallback(UUID id, TenantId tenantId, - CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap successMap, - ConcurrentMap failedMap, - ConcurrentMap firstExceptions) { + public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { this.id = id; this.tenantId = tenantId; - this.processingTimeoutLatch = processingTimeoutLatch; - this.ackMap = ackMap; - this.successMap = successMap; - this.failedMap = failedMap; - this.firstExceptions = firstExceptions; + this.ctx = ctx; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - T msg = ackMap.remove(id); - if (msg != null) { - successMap.put(id, msg); - } - if (msg != null && ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onSuccess(id); } @Override public void onFailure(RuleEngineException e) { log.trace("[{}] ON FAILURE", id, e); - T msg = ackMap.remove(id); - if (msg != null) { - failedMap.put(id, msg); - firstExceptions.putIfAbsent(tenantId, e); - } - if (ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onFailure(tenantId, id, e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 22f4edbbba..40017d2b40 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -19,7 +19,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; @@ -77,7 +76,7 @@ public class TbRuleEngineConsumerStats { public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) { int success = msg.getSuccessMap().size(); int pending = msg.getPendingMap().size(); - int failed = msg.getFailureMap().size(); + int failed = msg.getFailedMap().size(); totalMsgCounter.addAndGet(success + pending + failed); successMsgCounter.addAndGet(success); msg.getSuccessMap().values().forEach(m -> getTenantStats(m).logSuccess()); @@ -89,7 +88,7 @@ public class TbRuleEngineConsumerStats { msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTimeout()); } if (failed > 0) { - msg.getFailureMap().values().forEach(m -> getTenantStats(m).logFailed()); + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logFailed()); } failedIterationsCounter.incrementAndGet(); } else { @@ -103,7 +102,7 @@ public class TbRuleEngineConsumerStats { msg.getPendingMap().values().forEach(m -> getTenantStats(m).logTmpTimeout()); } if (failed > 0) { - msg.getFailureMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); + msg.getFailedMap().values().forEach(m -> getTenantStats(m).logTmpFailed()); } } msg.getExceptionsMap().forEach(tenantExceptions::putIfAbsent); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..bef733ec22 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngineSubmitStrategy { + + protected final String queueName; + protected List orderedMsgList; + private volatile boolean stopped; + + public AbstractTbRuleEngineSubmitStrategy(String queueName) { + this.queueName = queueName; + } + + protected abstract void doOnSuccess(UUID id); + + @Override + public void init(List> msgs) { + orderedMsgList = msgs.stream().map(msg -> new IdMsgPair(UUID.randomUUID(), msg)).collect(Collectors.toList()); + } + + @Override + public ConcurrentMap> getPendingMap() { + return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + List newOrderedMsgList = new ArrayList<>(reprocessMap.size()); + for (IdMsgPair pair : orderedMsgList) { + if (reprocessMap.containsKey(pair.uuid)) { + newOrderedMsgList.add(pair); + } + } + orderedMsgList = newOrderedMsgList; + } + + @Override + public void onSuccess(UUID id) { + if (!stopped) { + doOnSuccess(id); + } + } + + @Override + public void stop() { + stopped = true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..d0b1f7f99a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final int batchSize; + private final AtomicInteger packIdx = new AtomicInteger(0); + private final Map> pendingPack = new LinkedHashMap<>(); + private volatile BiConsumer> msgConsumer; + + public BatchTbRuleEngineSubmitStrategy(String queueName, int batchSize) { + super(queueName); + this.batchSize = batchSize; + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + packIdx.set(0); + } + + @Override + protected void doOnSuccess(UUID id) { + boolean endOfPendingPack; + synchronized (pendingPack) { + TbProtoQueueMsg msg = pendingPack.remove(id); + endOfPendingPack = msg != null && pendingPack.isEmpty(); + } + if (endOfPendingPack) { + packIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int startIdx = Math.min(packIdx.get() * batchSize, listSize); + int endIdx = Math.min(startIdx + batchSize, listSize); + synchronized (pendingPack) { + pendingPack.clear(); + for (int i = startIdx; i < endIdx; i++) { + IdMsgPair pair = orderedMsgList.get(i); + pendingPack.put(pair.uuid, pair.msg); + } + } + int submitSize = pendingPack.size(); + if (log.isInfoEnabled() && submitSize > 0) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + } + pendingPack.forEach(msgConsumer); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ffd1dd49d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + public BurstTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + } + orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); + } + + @Override + protected void doOnSuccess(UUID id) { + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java new file mode 100644 index 0000000000..2b2c203ec5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.UUID; + +public class IdMsgPair { + final UUID uuid; + final TbProtoQueueMsg msg; + + public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { + this.uuid = uuid; + this.msg = msg; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ae5993cb1c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,108 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +@Slf4j +public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private volatile BiConsumer> msgConsumer; + private volatile ConcurrentMap msgToEntityIdMap = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> entityIdToListMap = new ConcurrentHashMap<>(); + + public SequentialByEntityIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void init(List> msgs) { + super.init(msgs); + initMaps(); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + entityIdToListMap.forEach((entityId, queue) -> { + IdMsgPair msg = queue.peek(); + if (msg != null) { + msgConsumer.accept(msg.uuid, msg.msg); + } + }); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + initMaps(); + } + + @Override + protected void doOnSuccess(UUID id) { + EntityId entityId = msgToEntityIdMap.get(id); + if (entityId != null) { + Queue queue = entityIdToListMap.get(entityId); + if (queue != null) { + IdMsgPair next = null; + synchronized (queue) { + IdMsgPair expected = queue.peek(); + if (expected != null && expected.uuid.equals(id)) { + queue.poll(); + next = queue.peek(); + } + } + if (next != null) { + msgConsumer.accept(next.uuid, next.msg); + } + } + } + } + + private void initMaps() { + msgToEntityIdMap.clear(); + entityIdToListMap.clear(); + for (IdMsgPair pair : orderedMsgList) { + EntityId entityId = getEntityId(pair.msg.getValue()); + if (entityId != null) { + msgToEntityIdMap.put(pair.uuid, entityId); + entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); + } + } + } + + protected abstract EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..cd8a97e82c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByOriginatorIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@Slf4j +public class SequentialByOriginatorIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByOriginatorIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + try { + MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(msg.getTbMsg()); + return EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + } catch (InvalidProtocolBufferException e) { + log.warn("[{}] Failed to parse TbMsg: {}", queueName, msg); + return null; + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..b258c6db1b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialByEntityIdTbRuleEngineSubmitStrategy { + + public SequentialByTenantIdTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { + return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..ef45b983fc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +@Slf4j +public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { + + private final AtomicInteger msgIdx = new AtomicInteger(0); + private volatile BiConsumer> msgConsumer; + private volatile UUID expectedMsgId; + + public SequentialTbRuleEngineSubmitStrategy(String queueName) { + super(queueName); + } + + @Override + public void submitAttempt(BiConsumer> msgConsumer) { + this.msgConsumer = msgConsumer; + msgIdx.set(0); + submitNext(); + } + + @Override + public void update(ConcurrentMap> reprocessMap) { + super.update(reprocessMap); + } + + @Override + protected void doOnSuccess(UUID id) { + if (expectedMsgId.equals(id)) { + msgIdx.incrementAndGet(); + submitNext(); + } + } + + private void submitNext() { + int listSize = orderedMsgList.size(); + int idx = msgIdx.get(); + if (idx < listSize) { + IdMsgPair pair = orderedMsgList.get(idx); + expectedMsgId = pair.uuid; + if (log.isInfoEnabled()) { + log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + } + msgConsumer.accept(pair.uuid, pair.msg); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index e67936e294..8e0fcaa74a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.ProcessingAttemptContext; import java.util.UUID; import java.util.concurrent.ConcurrentMap; @@ -31,24 +32,27 @@ public class TbRuleEngineProcessingResult { @Getter private final boolean timeout; @Getter - private final ConcurrentMap> pendingMap; - @Getter - private final ConcurrentMap> successMap; - @Getter - private final ConcurrentMap> failureMap; - @Getter - private final ConcurrentMap exceptionsMap; + private final ProcessingAttemptContext ctx; - public TbRuleEngineProcessingResult(boolean timeout, - ConcurrentMap> pendingMap, - ConcurrentMap> successMap, - ConcurrentMap> failureMap, - ConcurrentMap exceptionsMap) { + public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { this.timeout = timeout; - this.pendingMap = pendingMap; - this.successMap = successMap; - this.failureMap = failureMap; - this.exceptionsMap = exceptionsMap; - this.success = !timeout && pendingMap.isEmpty() && failureMap.isEmpty(); + this.ctx = ctx; + this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); + } + + public ConcurrentMap> getPendingMap() { + return ctx.getPendingMap(); + } + + public ConcurrentMap> getSuccessMap() { + return ctx.getSuccessMap(); + } + + public ConcurrentMap> getFailedMap() { + return ctx.getFailedMap(); + } + + public ConcurrentMap getExceptionsMap() { + return ctx.getExceptionsMap(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index 1096bfbf7e..b6579b8dcb 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -16,12 +16,10 @@ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; -import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -77,10 +75,10 @@ public class TbRuleEngineProcessingStrategyFactory { return new TbRuleEngineProcessingDecision(true, null); } else { if (retryCount == 0) { - initialTotalCount = result.getPendingMap().size() + result.getFailureMap().size() + result.getSuccessMap().size(); + initialTotalCount = result.getPendingMap().size() + result.getFailedMap().size() + result.getSuccessMap().size(); } retryCount++; - double failedCount = result.getFailureMap().size() + result.getPendingMap().size(); + double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); @@ -90,7 +88,7 @@ public class TbRuleEngineProcessingStrategyFactory { } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); if (retryFailed) { - result.getFailureMap().forEach(toReprocess::put); + result.getFailedMap().forEach(toReprocess::put); } if (retryTimeout) { result.getPendingMap().forEach(toReprocess::put); @@ -125,7 +123,7 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailureMap().size(), result.getPendingMap().size()); + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java new file mode 100644 index 0000000000..7b22da97db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategy.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; + +public interface TbRuleEngineSubmitStrategy { + + void init(List> msgs); + + ConcurrentMap> getPendingMap(); + + void submitAttempt(BiConsumer> msgConsumer); + + void update(ConcurrentMap> reprocessMap); + + void onSuccess(UUID id); + + void stop(); +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java new file mode 100644 index 0000000000..f5a7457c17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.processing; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration; + +@Component +@Slf4j +public class TbRuleEngineSubmitStrategyFactory { + + public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) { + switch (configuration.getType()) { + case "BURST": + return new BurstTbRuleEngineSubmitStrategy(name); + case "BATCH": + return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); + case "SEQUENTIAL_WITHIN_ORIGINATOR": + return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL_WITHIN_TENANT": + return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); + case "SEQUENTIAL": + return new SequentialTbRuleEngineSubmitStrategy(name); + default: + throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!"); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index ce5065a64d..6d70056af0 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -123,7 +123,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId()); UUID requestId = request.getId(); localToDeviceRpcRequests.put(requestId, rpcMsg); - actorContext.getAppActor().tell(rpcMsg, ActorRef.noSender()); + actorContext.tell(rpcMsg, ActorRef.noSender()); scheduleToDeviceTimeout(request, requestId); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f08d2baf46..4c4955fea6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -578,27 +578,35 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: # TODO 2.5: specify correct ENV variable names. - name: "Main" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - name: "${TB_QUEUE_RULE_ENGINE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java index c1a8fd883d..0d21c59c9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueAckStrategyConfiguration.java @@ -20,13 +20,9 @@ import lombok.Data; @Data public class TbRuleEngineQueueAckStrategyConfiguration { -// @Value("${type}") private String type; -// @Value("${retries:3}") private int retries; -// @Value("${failure_percentage:0}") private double failurePercentage; -// @Value("${pause_between_retries:3}") private long pauseBetweenRetries; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index f89a615d7d..c5a24f20c6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -25,6 +25,7 @@ public class TbRuleEngineQueueConfiguration { private int pollInterval; private int partitions; private String packProcessingTimeout; - private TbRuleEngineQueueAckStrategyConfiguration ackStrategy; + private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; + private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java new file mode 100644 index 0000000000..e39dc0c322 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueSubmitStrategyConfiguration.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; + +@Data +public class TbRuleEngineQueueSubmitStrategyConfiguration { + + private String type; + private int batchSize; + +} From fcf436c6200db621545187be0338991c0809ee18 Mon Sep 17 00:00:00 2001 From: VoBa Date: Thu, 9 Apr 2020 18:40:34 +0300 Subject: [PATCH 049/196] [2.5] Added possibility to login by url params (#2581) * Added possibility to login by url params * Changes after code review --- ui/src/app/login/login.controller.js | 8 +++++++- ui/src/app/login/login.routes.js | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/src/app/login/login.controller.js b/ui/src/app/login/login.controller.js index 8dcd2a9d8e..d749ea0936 100644 --- a/ui/src/app/login/login.controller.js +++ b/ui/src/app/login/login.controller.js @@ -20,7 +20,7 @@ import logoSvg from '../../svg/logo_title_white.svg'; /* eslint-enable import/no-unresolved, import/default */ /*@ngInject*/ -export default function LoginController(toast, loginService, userService, types, $state/*, $rootScope, $log, $translate*/) { +export default function LoginController(toast, loginService, userService, types, $state, $stateParams/*, $rootScope, $log, $translate*/) { var vm = this; vm.logoSvg = logoSvg; @@ -32,6 +32,12 @@ export default function LoginController(toast, loginService, userService, types, vm.login = login; + if ($stateParams.username && $stateParams.password) { + vm.user.name = $stateParams.username; + vm.user.password = $stateParams.password; + doLogin(); + } + function doLogin() { loginService.login(vm.user).then(function success(response) { var token = response.data.token; diff --git a/ui/src/app/login/login.routes.js b/ui/src/app/login/login.routes.js index ac9401147b..51121a782a 100644 --- a/ui/src/app/login/login.routes.js +++ b/ui/src/app/login/login.routes.js @@ -25,7 +25,7 @@ import createPasswordTemplate from './create-password.tpl.html'; /*@ngInject*/ export default function LoginRoutes($stateProvider) { $stateProvider.state('login', { - url: '/login', + url: '/login?username&password', module: 'public', views: { "@": { From 7b3d47526792fd5a26fdaa9233afcc69f110ed5b Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Thu, 9 Apr 2020 18:41:37 +0300 Subject: [PATCH 050/196] [2.5] created rabbitmq queue (#2589) * created main classes for RabbitMq queue * created temp version rabbitmq * Merge branch 'develop/2.5' of https://github.com/thingsboard/thingsboard into develop/2.5-rabbitmq # Conflicts: # common/queue/pom.xml * rabbit improvements --- .../DefaultTbRuleEngineConsumerService.java | 2 + .../src/main/resources/thingsboard.yml | 10 + common/queue/pom.xml | 4 + .../queue/common/DefaultTbQueueMsg.java | 9 +- .../common/DefaultTbQueueRequestTemplate.java | 7 +- .../DefaultTbQueueResponseTemplate.java | 5 - .../RabbitMqMonolithQueueFactory.java | 128 +++++++++++++ .../provider/RabbitMqTbCoreQueueFactory.java | 117 ++++++++++++ .../RabbitMqTbRuleEngineQueueFactory.java | 100 ++++++++++ .../RabbitMqTransportQueueFactory.java | 97 ++++++++++ .../queue/rabbitmq/TbRabbitMqAdmin.java | 82 +++++++++ .../rabbitmq/TbRabbitMqConsumerTemplate.java | 173 ++++++++++++++++++ .../rabbitmq/TbRabbitMqProducerTemplate.java | 113 ++++++++++++ .../queue/rabbitmq/TbRabbitMqSettings.java | 65 +++++++ .../queue/sqs/TbAwsSqsProducerTemplate.java | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 10 + 16 files changed, 904 insertions(+), 20 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 38cc7b8dbd..c8f126b26c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -119,6 +119,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (submitExecutor != null) { submitExecutor.shutdownNow(); } + + ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); } @Override diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4c4955fea6..affd7e7908 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -550,6 +550,16 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/pom.xml b/common/queue/pom.xml index f55f69ece1..668da07e29 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -64,6 +64,10 @@ com.microsoft.azure azure-servicebus + + com.rabbitmq + amqp-client + org.springframework spring-context-support diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java index c7e439ef7d..7584e8c2d0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueMsg.java @@ -17,7 +17,6 @@ package org.thingsboard.server.queue.common; import lombok.Data; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import java.util.UUID; @@ -25,13 +24,7 @@ import java.util.UUID; public class DefaultTbQueueMsg implements TbQueueMsg { private final UUID key; private final byte[] data; - private DefaultTbQueueMsgHeaders headers; - - - public DefaultTbQueueMsg(UUID key, byte[] data) { - this.key = key; - this.data = data; - } + private final DefaultTbQueueMsgHeaders headers; public DefaultTbQueueMsg(TbQueueMsg msg) { this.key = msg.getKey(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 6ae9b9a33c..e8ebb934d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -20,7 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.errors.InterruptException; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueConsumer; @@ -28,7 +28,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.List; import java.util.UUID; @@ -128,10 +127,6 @@ public class DefaultTbQueueRequestTemplate> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java new file mode 100644 index 0000000000..3bf6d4667c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") +public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { + + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueTransportApiSettings transportApiSettings; + private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin admin; + + public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.rabbitMqSettings = rabbitMqSettings; + this.coreSettings = coreSettings; + this.transportApiSettings = transportApiSettings; + this.ruleEngineSettings = ruleEngineSettings; + this.partitionService = partitionService; + this.serviceInfoProvider = serviceInfoProvider; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToCoreMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createTransportApiResponseProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java new file mode 100644 index 0000000000..52d28cccd7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") +public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { + + private final PartitionService partitionService; + private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueAdmin admin; + + public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, + TbRabbitMqSettings rabbitMqSettings, + TbQueueAdmin admin) { + this.partitionService = partitionService; + this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.ruleEngineSettings = ruleEngineSettings; + this.rabbitMqSettings = rabbitMqSettings; + this.admin = admin; + } + + @Override + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java new file mode 100644 index 0000000000..dc6154255c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") +@Slf4j +public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { + private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueAdmin admin; + private final TbServiceInfoProvider serviceInfoProvider; + + public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbRabbitMqSettings rabbitMqSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { + this.transportApiSettings = transportApiSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.rabbitMqSettings = rabbitMqSettings; + this.admin = admin; + this.serviceInfoProvider = serviceInfoProvider; + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { + TbRabbitMqProducerTemplate> producerTemplate = + new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + + TbRabbitMqConsumerTemplate> consumerTemplate = + new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); + templateBuilder.queueAdmin(admin); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); + templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); + templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); + templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); + return templateBuilder.build(); + } + + @Override + public TbQueueProducer> createRuleEngineMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreMsgProducer() { + return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + } + + @Override + public TbQueueConsumer> createTransportNotificationsConsumer() { + return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java new file mode 100644 index 0000000000..fbd678045b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.TbQueueAdmin; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +@Slf4j +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqAdmin implements TbQueueAdmin { + + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) { + this.rabbitMqSettings = rabbitMqSettings; + + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void createTopicIfNotExists(String topic) { + try { + channel.queueDeclare(topic, false, false, false, null); + } catch (IOException e) { + log.error("Failed to bind queue: [{}]", topic, e); + } + } + + @PreDestroy + private void destroy() { + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close Chanel.", e); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close Connection.", e); + } + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java new file mode 100644 index 0000000000..25d7719163 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.GetResponse; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@Slf4j +public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { + + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final String topic; + private final TbQueueMsgDecoder decoder; + private final TbRabbitMqSettings rabbitMqSettings; + private final Channel channel; + private final Connection connection; + + private volatile Set partitions; + private volatile boolean subscribed; + private volatile Set queues; + private volatile boolean stopped; + + public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + this.admin = admin; + this.decoder = decoder; + this.topic = topic; + this.rabbitMqSettings = rabbitMqSettings; + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + stopped = false; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public void subscribe() { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } + + @Override + public void subscribe(Set partitions) { + this.partitions = partitions; + subscribed = false; + } + + @Override + public void unsubscribe() { + stopped = true; + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + if (!subscribed) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + + queues.forEach(admin::createTopicIfNotExists); + subscribed = true; + } + + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).map(message -> { + try { + return decode(message); + } catch (InvalidProtocolBufferException e) { + log.error("Failed to decode message: [{}].", message); + throw new RuntimeException("Failed to decode message.", e); + } + }).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } + } + + public T decode(GetResponse message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); + return decoder.decode(msg); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java new file mode 100644 index 0000000000..91b46213a5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java @@ -0,0 +1,113 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.DefaultTbQueueMsg; + +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class TbRabbitMqProducerTemplate implements TbQueueProducer { + private final String defaultTopic; + private final Gson gson = new Gson(); + private final TbQueueAdmin admin; + private final TbRabbitMqSettings rabbitMqSettings; + private ListeningExecutorService producerExecutor; + private final Channel channel; + private final Connection connection; + + public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { + this.admin = admin; + this.defaultTopic = defaultTopic; + this.rabbitMqSettings = rabbitMqSettings; + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + try { + connection = rabbitMqSettings.getConnectionFactory().newConnection(); + } catch (IOException | TimeoutException e) { + log.error("Failed to create connection.", e); + throw new RuntimeException("Failed to create connection.", e); + } + + try { + channel = connection.createChannel(); + } catch (IOException e) { + log.error("Failed to create chanel.", e); + throw new RuntimeException("Failed to create chanel.", e); + } + } + + @Override + public void init() { + + } + + @Override + public String getDefaultTopic() { + return defaultTopic; + } + + @Override + public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + AMQP.BasicProperties properties = new AMQP.BasicProperties(); + try { + channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); + if (callback != null) { + callback.onSuccess(null); + } + } catch (IOException e) { + log.error("Failed publish message: [{}].", msg, e); + if (callback != null) { + callback.onFailure(e); + } + } + } + + @Override + public void stop() { + if (producerExecutor != null) { + producerExecutor.shutdownNow(); + } + if (channel != null) { + try { + channel.close(); + } catch (IOException | TimeoutException e) { + log.error("Failed to close the channel."); + } + } + if (connection != null) { + try { + connection.close(); + } catch (IOException e) { + log.error("Failed to close the connection."); + } + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java new file mode 100644 index 0000000000..e0156e6dc8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import com.rabbitmq.client.ConnectionFactory; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +@Component +@Data +public class TbRabbitMqSettings { + @Value("${queue.rabbitmq.exchange_name:}") + private String exchangeName; + @Value("${queue.rabbitmq.host:}") + private String host; + @Value("${queue.rabbitmq.port:}") + private int port; + @Value("${queue.rabbitmq.virtual_host:}") + private String virtualHost; + @Value("${queue.rabbitmq.username:}") + private String username; + @Value("${queue.rabbitmq.password:}") + private String password; + @Value("${queue.rabbitmq.automatic_recovery_enabled:}") + private boolean automaticRecoveryEnabled; + @Value("${queue.rabbitmq.connection_timeout:}") + private int connectionTimeout; + @Value("${queue.rabbitmq.handshake_timeout:}") + private int handshakeTimeout; + + private ConnectionFactory connectionFactory; + + @PostConstruct + private void init() { + connectionFactory = new ConnectionFactory(); + connectionFactory.setHost(host); + connectionFactory.setPort(port); + connectionFactory.setVirtualHost(virtualHost); + connectionFactory.setUsername(username); + connectionFactory.setPassword(password); + connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); + connectionFactory.setConnectionTimeout(connectionTimeout); + connectionFactory.setHandshakeTimeout(handshakeTimeout); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 6eed92d296..2d85539184 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -78,7 +78,7 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { SendMessageRequest sendMsgRequest = new SendMessageRequest(); sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg.getKey(), msg.getData()))); + sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); sendMsgRequest.withMessageGroupId(msg.getKey().toString()); ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 9b04867cfe..395f1d09e0 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -88,6 +88,16 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 1dd3334825212603da640af693f1f95e274943c5 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 9 Apr 2020 14:44:35 +0300 Subject: [PATCH 051/196] moved jsinvoke.proto to queue, fixed js-executor, added createRemoteJsRequestTemplate to RuleEngine and Core factories --- .../service/script/RemoteJsInvokeService.java | 67 +++++-------------- .../queue/common/TbProtoJsQueueMsg.java | 43 ++++++++++++ .../server/queue/common/TbProtoQueueMsg.java | 2 +- .../provider/AwsSqsMonolithQueueFactory.java | 8 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 8 +++ .../AwsSqsTbRuleEngineQueueFactory.java | 8 +++ .../InMemoryMonolithQueueFactory.java | 8 +++ .../provider/KafkaMonolithQueueFactory.java | 48 ++++++++++++- .../provider/KafkaTbCoreQueueFactory.java | 48 ++++++++++++- .../KafkaTbRuleEngineQueueFactory.java | 48 ++++++++++++- .../provider/PubSubMonolithQueueFactory.java | 8 +++ .../provider/PubSubTbCoreQueueFactory.java | 8 +++ .../PubSubTbRuleEngineQueueFactory.java | 12 +++- .../ServiceBusMonolithQueueFactory.java | 8 +++ ...java => ServiceBusTbCoreQueueFactory.java} | 24 ++++--- .../ServiceBusTbRuleEngineQueueFactory.java | 8 +++ .../queue/provider/TbCoreQueueFactory.java | 14 ++-- .../provider/TbRuleEngineQueueFactory.java | 12 ++-- .../TbQueueRemoteJsInvokeSettings.java | 42 ++++++++++++ .../queue}/src/main/proto/jsinvoke.proto | 0 .../api/jsInvokeMessageProcessor.js | 5 +- 21 files changed, 353 insertions(+), 76 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java rename common/queue/src/main/java/org/thingsboard/server/queue/provider/{ServiceBusTbCoreQueueProvider.java => ServiceBusTbCoreQueueFactory.java} (84%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java rename {application => common/queue}/src/main/proto/jsinvoke.proto (100%) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index b0554decea..8d1f9d662a 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -21,14 +21,15 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -41,28 +42,13 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "remote", matchIfMissing = true) +@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')") @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Value("${js.remote.request_topic}") - private String requestTopic; - - @Value("${js.remote.response_topic_prefix}") - private String responseTopicPrefix; - - @Value("${js.remote.max_pending_requests}") - private long maxPendingRequests; - @Value("${js.remote.max_requests_timeout}") private long maxRequestsTimeout; - @Value("${js.remote.response_poll_interval}") - private int responsePollDuration; - - @Value("${js.remote.response_auto_commit_interval}") - private int autoCommitInterval; - @Getter @Value("${js.remote.max_errors}") private int maxErrors; @@ -94,43 +80,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { } } - private DefaultTbQueueRequestTemplate, TbProtoQueueMsg> defaultTemplate; + @Autowired + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + private Map scriptIdToBodysMap = new ConcurrentHashMap<>(); @PostConstruct public void init() { -// TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); -// requestBuilder.settings(kafkaSettings); -// requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId()); -// requestBuilder.defaultTopic(requestTopic); -// requestBuilder.encoder(new RemoteJsRequestEncoder()); - TbQueueProducer> producer; - -// TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder responseBuilder = TBKafkaConsumerTemplate.builder(); -// responseBuilder.settings(kafkaSettings); -// responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId()); -// responseBuilder.clientId("js-" + nodeIdProvider.getNodeId()); -// responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); -// responseBuilder.decoder(new RemoteJsResponseDecoder()); -// responseBuilder.requestIdExtractor((response) -> new UUID(response.getRequestIdMSB(), response.getRequestIdLSB())); -// -// TbKafkaRequestTemplate.TbKafkaRequestTemplateBuilder -// builder = TbKafkaRequestTemplate.builder(); -// builder.requestTemplate(requestBuilder.build()); -// builder.responseTemplate(responseBuilder.build()); -// builder.maxPendingRequests(maxPendingRequests); -// builder.maxRequestTimeout(maxRequestsTimeout); -// builder.pollInterval(responsePollDuration); -// defaultTemplate = builder.build(); -// defaultTemplate.init(); + requestTemplate.init(); } @PreDestroy public void destroy() { - if (defaultTemplate != null) { - defaultTemplate.stop(); + if (requestTemplate != null) { + requestTemplate.stop(); } } @@ -147,7 +110,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); log.trace("Post compile request for scriptId [{}]", scriptId); - ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); Futures.addCallback(future, new FutureCallback>() { @@ -199,7 +162,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setInvokeRequest(jsRequestBuilder.build()) .build(); - ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); kafkaPushedMsgs.incrementAndGet(); Futures.addCallback(future, new FutureCallback>() { @Override @@ -239,7 +202,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setReleaseRequest(jsRequest) .build(); - ListenableFuture> future = defaultTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java new file mode 100644 index 0000000000..07417c4a9e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoJsQueueMsg.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.thingsboard.server.queue.TbQueueMsgHeaders; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +public class TbProtoJsQueueMsg extends TbProtoQueueMsg { + + public TbProtoJsQueueMsg(UUID key, T value) { + super(key, value); + } + + public TbProtoJsQueueMsg(UUID key, T value, TbQueueMsgHeaders headers) { + super(key, value, headers); + } + + @Override + public byte[] getData() { + try { + return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java index 8823002530..2eb76950dc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -25,7 +25,7 @@ import java.util.UUID; public class TbProtoQueueMsg implements TbQueueMsg { private final UUID key; - private final T value; + protected final T value; private final TbQueueMsgHeaders headers; public TbProtoQueueMsg(UUID key, T value) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 9f70345301..26da11aa3b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -18,10 +18,13 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -125,4 +128,9 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 770e7fa65c..87aba51843 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,6 +29,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -114,4 +117,9 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index f87f108453..d83b760c19 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,6 +26,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -96,4 +99,9 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a23c93e015..baad96182f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,6 +28,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -110,4 +113,9 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index fa36f8284b..dae4221e91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,18 +30,25 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -50,13 +60,15 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,6 +76,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -175,4 +188,37 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); return requestBuilder.build(); } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 5e46d67eed..e148416bb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,16 +30,23 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-core'") public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @@ -47,18 +57,21 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings) { + TbQueueTransportApiSettings transportApiSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -148,4 +161,37 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index eef10f35dc..8262d27354 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,16 +28,23 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-rule-engine'") public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -44,16 +54,19 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings) { + TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -124,4 +137,37 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); return consumerBuilder.build(); } + + @Override + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + + TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + responseBuilder.settings(kafkaSettings); + responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); + responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); + responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); +// responseBuilder.autoCommit(true); +// responseBuilder.autoCommitIntervalMs(autoCommitInterval); + responseBuilder.decoder(msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + } + ); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.requestTemplate(requestBuilder.build()); + builder.responseTemplate(responseBuilder.build()); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index f00bd7caef..b1afc61dbd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,6 +29,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -133,4 +136,9 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index da2fe3acdf..edfcdc3d8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,6 +28,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -110,4 +113,9 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index da65d583c1..101f335536 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,15 +26,17 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @Component @@ -99,4 +102,9 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 914e200fc9..8b01c38607 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,9 +29,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -131,4 +134,9 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul public TbQueueProducer> createTransportApiResponseProducer() { return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java similarity index 84% rename from common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java rename to common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 334af58133..523ac88bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -27,9 +28,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -39,7 +42,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") -public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { +public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbServiceBusSettings serviceBusSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; @@ -49,13 +52,13 @@ public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueAdmin admin; - public ServiceBusTbCoreQueueProvider(TbServiceBusSettings serviceBusSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueTransportApiSettings transportApiSettings, + TbQueueRuleEngineSettings ruleEngineSettings, + PartitionService partitionService, + TbServiceInfoProvider serviceInfoProvider, + TbQueueAdmin admin) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -113,4 +116,9 @@ public class ServiceBusTbCoreQueueProvider implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 3ceb837606..8a2f9b396b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -25,9 +26,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -96,4 +99,9 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 1a3ba1ab98..59a8f449b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.gen.transport.TransportProtos.*; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; /** * Responsible for initialization of various Producers and Consumers used by TB Core Node. @@ -94,5 +98,5 @@ public interface TbCoreQueueFactory { */ TbQueueProducer> createTransportApiResponseProducer(); - + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 300fb986a9..561b3e84ea 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; /** @@ -82,4 +85,5 @@ public interface TbRuleEngineQueueFactory { */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); + TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java new file mode 100644 index 0000000000..cdad407b19 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueRemoteJsInvokeSettings { + @Value("${js.remote.request_topic}") + private String requestTopic; + + @Value("${js.remote.response_topic_prefix}") + private String responseTopic; + + @Value("${js.remote.max_pending_requests}") + private long maxPendingRequests; + + @Value("${js.remote.response_poll_interval}") + private int responsePollInterval; + + @Value("${js.remote.response_auto_commit_interval}") + private int autoCommitInterval; + + @Value("${js.remote.max_requests_timeout}") + private long maxRequestsTimeout; +} diff --git a/application/src/main/proto/jsinvoke.proto b/common/queue/src/main/proto/jsinvoke.proto similarity index 100% rename from application/src/main/proto/jsinvoke.proto rename to common/queue/src/main/proto/jsinvoke.proto diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index c17b1ddde4..f0facf8cc1 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,6 +19,7 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; +let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -43,6 +44,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var responseTopic; try { var request = JSON.parse(message.value.toString('utf8')); + headers = message.headers; var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); buf = message.headers['responseTopic']; @@ -148,7 +150,8 @@ JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseT messages: [ { key: scriptId, - value: rawResponse + value: rawResponse, + headers: headers } ] } From 3881dc8423623e636d5c10b50160373b08a9433a Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Fri, 10 Apr 2020 15:04:24 +0300 Subject: [PATCH 052/196] Add new alias Current tenant (#2596) --- ui/src/app/api/entity.service.js | 5 +++++ ui/src/app/common/types.constant.js | 7 ++++++- ui/src/app/entity/entity-select.directive.js | 6 ++++-- ui/src/app/entity/entity-select.tpl.html | 4 ++-- ui/src/app/locale/locale.constant-cs_CZ.json | 1 + ui/src/app/locale/locale.constant-en_US.json | 1 + ui/src/app/locale/locale.constant-ru_RU.json | 1 + ui/src/app/locale/locale.constant-uk_UA.json | 1 + 8 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ui/src/app/api/entity.service.js b/ui/src/app/api/entity.service.js index b795432dee..f692c99051 100644 --- a/ui/src/app/api/entity.service.js +++ b/ui/src/app/api/entity.service.js @@ -435,6 +435,10 @@ function EntityService($http, $q, $filter, $translate, $log, userService, device if (user.authority === 'CUSTOMER_USER') { entityId.id = user.customerId; } + } else if (entityType === types.aliasEntityType.current_tenant){ + let user = userService.getCurrentUser(); + entityId.entityType = types.entityType.tenant; + entityId.id = user.tenantId; } return entityId; } @@ -806,6 +810,7 @@ function EntityService($http, $q, $filter, $translate, $log, userService, device entityTypes.dashboard = types.entityType.dashboard; if (useAliasEntityTypes) { entityTypes.current_customer = types.aliasEntityType.current_customer; + entityTypes.current_tenant = types.aliasEntityType.current_tenant; } break; case 'CUSTOMER_USER': diff --git a/ui/src/app/common/types.constant.js b/ui/src/app/common/types.constant.js index 86bf952582..aff3821573 100644 --- a/ui/src/app/common/types.constant.js +++ b/ui/src/app/common/types.constant.js @@ -408,7 +408,8 @@ export default angular.module('thingsboard.types', []) } }, aliasEntityType: { - current_customer: "CURRENT_CUSTOMER" + current_customer: "CURRENT_CUSTOMER", + current_tenant: "CURRENT_TENANT" }, entityTypeTranslations: { "DEVICE": { @@ -474,6 +475,10 @@ export default angular.module('thingsboard.types', []) "CURRENT_CUSTOMER": { type: 'entity.type-current-customer', list: 'entity.type-current-customer' + }, + "CURRENT_TENANT": { + type: 'entity.type-current-tenant', + list: 'entity.type-current-tenant' } }, entityField: { diff --git a/ui/src/app/entity/entity-select.directive.js b/ui/src/app/entity/entity-select.directive.js index 9b1c763a60..e93a769eec 100644 --- a/ui/src/app/entity/entity-select.directive.js +++ b/ui/src/app/entity/entity-select.directive.js @@ -22,13 +22,14 @@ import entitySelectTemplate from './entity-select.tpl.html'; /* eslint-enable import/no-unresolved, import/default */ /*@ngInject*/ -export default function EntitySelect($compile, $templateCache, entityService) { +export default function EntitySelect($compile, $templateCache, entityService, types) { var linker = function (scope, element, attrs, ngModelCtrl) { var template = $templateCache.get(entitySelectTemplate); element.html(template); scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; + scope.entityTypeCurrentTenant = types.aliasEntityType.current_tenant; var entityTypes = entityService.prepareAllowedEntityTypesList(scope.allowedEntityTypes, scope.useAliasEntityTypes); @@ -48,7 +49,8 @@ export default function EntitySelect($compile, $templateCache, entityService) { scope.updateView = function () { if (!scope.disabled) { var value = ngModelCtrl.$viewValue; - if (scope.model && scope.model.entityType && scope.model.entityId) { + if (scope.model && scope.model.entityType && + (scope.model.entityId || scope.model.entityType === scope.entityTypeCurrentTenant)) { if (!value) { value = {}; } diff --git a/ui/src/app/entity/entity-select.tpl.html b/ui/src/app/entity/entity-select.tpl.html index 59133d541d..8800f9760f 100644 --- a/ui/src/app/entity/entity-select.tpl.html +++ b/ui/src/app/entity/entity-select.tpl.html @@ -25,11 +25,11 @@ allowed-entity-types="allowedEntityTypes" ng-model="model.entityType"> - - \ No newline at end of file + diff --git a/ui/src/app/locale/locale.constant-cs_CZ.json b/ui/src/app/locale/locale.constant-cs_CZ.json index 0006afc74b..2a297cd9bb 100644 --- a/ui/src/app/locale/locale.constant-cs_CZ.json +++ b/ui/src/app/locale/locale.constant-cs_CZ.json @@ -770,6 +770,7 @@ "list-of-rulenodes": "{ count, plural, 1 {Jeden uzel pravidla} other {Seznam # uzlů pravidel} }", "rulenode-name-starts-with": "Uzly pravidel, jejichž název začíná '{{prefix}}'", "type-current-customer": "Stávající zákazník", + "type-current-tenant": "Stávající tenant", "search": "Vyhledat entity", "selected-entities": "{ count, plural, 1 {1 entita} other {# entit} } zvoleno", "entity-name": "Název entity", diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index a0750164ca..d69778a787 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -811,6 +811,7 @@ "list-of-rulenodes": "{ count, plural, 1 {One rule node} other {List of # rule nodes} }", "rulenode-name-starts-with": "Rule nodes whose names start with '{{prefix}}'", "type-current-customer": "Current Customer", + "type-current-tenant": "Current Tenant", "search": "Search entities", "selected-entities": "{ count, plural, 1 {1 entity} other {# entities} } selected", "entity-name": "Entity name", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index 47c70da48a..f86eb988ff 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -809,6 +809,7 @@ "list-of-rulenodes": "{ count, plural, 1 {Одно правило} other {Список из # правил} }", "rulenode-name-starts-with": "Правила, чьи названия начинаются с '{{prefix}}'", "type-current-customer": "Текущий клиент", + "type-current-tenant": "Текущий владелец", "search": "Поиск объектов", "selected-entities": "Выбран(ы) { count, plural, 1 {1 объект} few {# объекта} other {# объектов} }", "entity-name": "Название объекта", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index 7b882423ed..5a0899014d 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -942,6 +942,7 @@ "list-of-rulenodes": "{ count, plural, 1 {Одне правило} other {Список # правил} }", "rulenode-name-starts-with": "Список правил, імена яких починаються '{{prefix}}'", "type-current-customer": "Поточний клієнт", + "type-current-tenant": "Поточний власник", "search": "Пошук сутностей", "selected-entities": "{ count, plural, 1 {1 сутність} other {# сутності} } вибрано", "entity-name": "Ім'я сутності", From e8fffdbc4d2b6b921ce311c2399b1a16dcbf38c4 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Fri, 10 Apr 2020 15:04:34 +0300 Subject: [PATCH 053/196] Fix manage dashboard states for Safari (#2602) --- ui/src/app/dashboard/states/manage-dashboard-states.tpl.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/app/dashboard/states/manage-dashboard-states.tpl.html b/ui/src/app/dashboard/states/manage-dashboard-states.tpl.html index 23b2433d0a..5c8f363003 100644 --- a/ui/src/app/dashboard/states/manage-dashboard-states.tpl.html +++ b/ui/src/app/dashboard/states/manage-dashboard-states.tpl.html @@ -15,7 +15,7 @@ limitations under the License. --> - +
@@ -72,7 +72,7 @@ - + From 39ba550d411181d8025ce20f33cc4cf6c5b47888 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Fri, 10 Apr 2020 15:04:45 +0300 Subject: [PATCH 054/196] Fix not add entityLabel for header action open dashboard state (#2601) --- ui/src/app/components/widget/widget.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/app/components/widget/widget.controller.js b/ui/src/app/components/widget/widget.controller.js index 5346246eae..83225edcc2 100644 --- a/ui/src/app/components/widget/widget.controller.js +++ b/ui/src/app/components/widget/widget.controller.js @@ -152,7 +152,7 @@ export default function WidgetController($scope, $state, $timeout, $window, $ocL var entityInfo = getActiveEntityInfo(); var entityId = entityInfo ? entityInfo.entityId : null; var entityName = entityInfo ? entityInfo.entityName : null; - var entityLabel = entityInfo && entityInfo.label ? entityInfo.label : null; + var entityLabel = entityInfo && entityInfo.entityLabel ? entityInfo.entityLabel : null; handleWidgetAction($event, this.descriptor, entityId, entityName, null, entityLabel); } widgetContext.customHeaderActions.push(headerAction); From 2a1c55f02217e72eecd955c16c8dbcab7ac74ede Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Fri, 10 Apr 2020 15:04:55 +0300 Subject: [PATCH 055/196] [2.5] Revert change JSON form (#2598) * Revert change * Revert file --- .../components/react/json-form-ace-editor.jsx | 6 ++--- .../app/components/react/json-form-array.jsx | 2 +- .../components/react/json-form-fieldset.jsx | 2 +- .../react/json-form-schema-form.jsx | 26 +++++++------------ ui/src/app/components/react/json-form.scss | 3 --- 5 files changed, 15 insertions(+), 24 deletions(-) diff --git a/ui/src/app/components/react/json-form-ace-editor.jsx b/ui/src/app/components/react/json-form-ace-editor.jsx index 966419b3fe..2329bbddf7 100644 --- a/ui/src/app/components/react/json-form-ace-editor.jsx +++ b/ui/src/app/components/react/json-form-ace-editor.jsx @@ -83,9 +83,9 @@ class ThingsboardAceEditor extends React.Component { fixAceEditor(editor); } - onToggleFull(groupId) { + onToggleFull() { this.setState({ isFull: !this.state.isFull }); - this.props.onToggleFullscreen(groupId); + this.props.onToggleFullscreen(); this.updateAceEditorSize = true; } @@ -140,7 +140,7 @@ class ThingsboardAceEditor extends React.Component {
- this.onToggleFull(this.props.groupId)}/> +
diff --git a/ui/src/app/components/react/json-form-fieldset.jsx b/ui/src/app/components/react/json-form-fieldset.jsx index 4e078d72e2..f668ba774c 100644 --- a/ui/src/app/components/react/json-form-fieldset.jsx +++ b/ui/src/app/components/react/json-form-fieldset.jsx @@ -19,7 +19,7 @@ class ThingsboardFieldSet extends React.Component { render() { let forms = this.props.form.items.map(function(form, index){ - return this.props.builder(form, this.props.groupId, this.props.model, index, this.props.onChange, this.props.onColorClick, this.props.onIconClick, this.props.onToggleFullscreen, this.props.mapper, this.props.builder); + return this.props.builder(form, this.props.model, index, this.props.onChange, this.props.onColorClick, this.props.onIconClick, this.props.onToggleFullscreen, this.props.mapper, this.props.builder); }.bind(this)); return ( diff --git a/ui/src/app/components/react/json-form-schema-form.jsx b/ui/src/app/components/react/json-form-schema-form.jsx index 2172ac3640..04970f601b 100644 --- a/ui/src/app/components/react/json-form-schema-form.jsx +++ b/ui/src/app/components/react/json-form-schema-form.jsx @@ -40,9 +40,6 @@ class ThingsboardSchemaForm extends React.Component { constructor(props) { super(props); - this.state = { - groupId: null, - }; this.mapper = { 'number': ThingsboardNumber, @@ -88,15 +85,12 @@ class ThingsboardSchemaForm extends React.Component { this.props.onIconClick(event); } - onToggleFullscreen(groupId) { - this.setState({ - groupId: groupId - }); + onToggleFullscreen() { this.props.onToggleFullscreen(); } - builder(form, groupId, model, index, onChange, onColorClick, onIconClick, onToggleFullscreen, mapper) { + builder(form, model, index, onChange, onColorClick, onIconClick, onToggleFullscreen, mapper) { var type = form.type; let Field = this.mapper[type]; if(!Field) { @@ -109,21 +103,21 @@ class ThingsboardSchemaForm extends React.Component { return null; } } - return + return } - createSchema(theForm, groupId) { + createSchema(theForm) { let merged = utils.merge(this.props.schema, theForm, this.props.ignore, this.props.option); let mapper = this.mapper; if(this.props.mapper) { mapper = _.merge(this.mapper, this.props.mapper); } let forms = merged.map(function(form, index) { - return this.builder(form, groupId, this.props.model, index, this.onChange, this.onColorClick, this.onIconClick, this.onToggleFullscreen, mapper); + return this.builder(form, this.props.model, index, this.onChange, this.onColorClick, this.onIconClick, this.onToggleFullscreen, mapper); }.bind(this)); let formClass = 'SchemaForm'; - if (this.props.isFullscreen && groupId === this.state.groupId) { + if (this.props.isFullscreen) { formClass += ' SchemaFormFullscreen'; } @@ -136,7 +130,7 @@ class ThingsboardSchemaForm extends React.Component { if(this.props.groupInfoes&&this.props.groupInfoes.length>0){ let content=[]; for(let info of this.props.groupInfoes){ - let forms = this.createSchema(this.props.form[info.formIndex], info.formIndex); + let forms = this.createSchema(this.props.form[info.formIndex]); let item = ; content.push(item); } @@ -166,8 +160,8 @@ class ThingsboardSchemaGroup extends React.Component{ render() { let theCla = "pull-right fa fa-chevron-down md-toggle-icon"+(this.state.showGroup?"":" tb-toggled") return (
-
{this.props.info.GroupTitle}
-
{this.props.forms}
-
); +
{this.props.info.GroupTitle}
+
{this.props.forms}
+ ); } } diff --git a/ui/src/app/components/react/json-form.scss b/ui/src/app/components/react/json-form.scss index 91c14d51ff..85825abf42 100644 --- a/ui/src/app/components/react/json-form.scss +++ b/ui/src/app/components/react/json-form.scss @@ -24,15 +24,12 @@ $input-label-float-scale: .75 !default; .tb-fullscreen { [name="ReactSchemaForm"] { .SchemaForm { - display: none; - &.SchemaFormFullscreen { position: absolute; top: 0; right: 0; bottom: 0; left: 0; - display: block; > div:not(.fullscreen-form-field) { display: none !important; From a3a17dc8282a2a955f1e0b027234367339a2f1a7 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Fri, 10 Apr 2020 15:06:44 +0300 Subject: [PATCH 056/196] add sendActivationEmail as request param (default -> true) to activate method in auth controller and login service (#2597) --- .../thingsboard/server/controller/AuthController.java | 11 +++++++---- ui/src/app/api/login.service.js | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 0cb9b29501..44449eb3d6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -199,6 +199,7 @@ public class AuthController extends BaseController { @ResponseBody public JsonNode activateUser( @RequestBody JsonNode activateRequest, + @RequestParam(required = false, defaultValue = "true") boolean sendActivationMail, HttpServletRequest request) throws ThingsboardException { try { String activateToken = activateRequest.get("activateToken").asText(); @@ -213,10 +214,12 @@ public class AuthController extends BaseController { String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); - try { - mailService.sendAccountActivatedEmail(loginUrl, email); - } catch (Exception e) { - log.info("Unable to send account activation email [{}]", e.getMessage()); + if (sendActivationMail) { + try { + mailService.sendAccountActivatedEmail(loginUrl, email); + } catch (Exception e) { + log.info("Unable to send account activation email [{}]", e.getMessage()); + } } JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); diff --git a/ui/src/app/api/login.service.js b/ui/src/app/api/login.service.js index 2322c13b63..0707070ed5 100644 --- a/ui/src/app/api/login.service.js +++ b/ui/src/app/api/login.service.js @@ -85,9 +85,12 @@ function LoginService($http, $q) { return deferred.promise; } - function activate(activateToken, password) { + function activate(activateToken, password, sendActivationMail) { var deferred = $q.defer(); var url = '/api/noauth/activate'; + if(sendActivationMail === true || sendActivationMail === false) { + url += '?sendActivationMail=' + sendActivationMail; + } $http.post(url, {activateToken: activateToken, password: password}).then(function success(response) { deferred.resolve(response); }, function fail() { From 8442159811021264cea5c535aec3a57661bb9b52 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 13 Apr 2020 09:34:35 +0300 Subject: [PATCH 057/196] [2.5] vulnerabilities (#2608) * Version updates * added @Retention(RetentionPolicy.RUNTIME) for all custom Dao annotations. * changed getId, setId to getUuid, setUuid from BaseEntity, improvement AbstractControllerTest * fix rabbitmq * fix RuleChainManagerActor * refactored InMemory queue Co-authored-by: Andrii Shvaika --- .../ruleChain/RuleChainManagerActor.java | 4 +- .../actors/service/ContextAwareActor.java | 4 +- .../controller/AbstractControllerTest.java | 4 +- .../thingsboard/server/dao/util/HsqlDao.java | 4 ++ .../server/dao/util/NoSqlAnyDao.java | 4 ++ .../thingsboard/server/dao/util/NoSqlDao.java | 4 ++ .../server/dao/util/NoSqlTsDao.java | 4 ++ .../thingsboard/server/dao/util/PsqlDao.java | 4 ++ .../server/dao/util/PsqlTsAnyDao.java | 4 ++ .../thingsboard/server/dao/util/SqlDao.java | 4 ++ .../server/dao/util/SqlTsAnyDao.java | 4 ++ .../thingsboard/server/dao/util/SqlTsDao.java | 4 ++ .../server/dao/util/TimescaleDBTsDao.java | 4 ++ .../server/queue/memory/InMemoryStorage.java | 20 +++--- .../queue/memory/InMemoryTbQueueConsumer.java | 38 +++++++++-- .../queue/memory/InMemoryTbQueueProducer.java | 2 +- .../InMemoryMonolithQueueFactory.java | 66 ++++++++++--------- .../InMemoryTbTransportQueueFactory.java | 42 ++++++------ .../RabbitMqMonolithQueueFactory.java | 8 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 8 +++ .../RabbitMqTbRuleEngineQueueFactory.java | 8 +++ .../CassandraBaseComponentDescriptorDao.java | 6 +- .../dao/event/CassandraBaseEventDao.java | 6 +- .../server/dao/model/BaseEntity.java | 4 +- .../server/dao/model/BaseSqlEntity.java | 5 +- .../dao/model/nosql/AdminSettingsEntity.java | 4 +- .../server/dao/model/nosql/AlarmEntity.java | 4 +- .../server/dao/model/nosql/AssetEntity.java | 4 +- .../dao/model/nosql/AuditLogEntity.java | 4 +- .../nosql/ComponentDescriptorEntity.java | 4 +- .../dao/model/nosql/CustomerEntity.java | 4 +- .../dao/model/nosql/DashboardEntity.java | 4 +- .../dao/model/nosql/DashboardInfoEntity.java | 4 +- .../model/nosql/DeviceCredentialsEntity.java | 4 +- .../server/dao/model/nosql/DeviceEntity.java | 4 +- .../dao/model/nosql/EntityViewEntity.java | 10 +++ .../server/dao/model/nosql/EventEntity.java | 4 +- .../dao/model/nosql/RuleChainEntity.java | 4 +- .../dao/model/nosql/RuleNodeEntity.java | 4 +- .../server/dao/model/nosql/TenantEntity.java | 4 +- .../model/nosql/UserCredentialsEntity.java | 4 +- .../server/dao/model/nosql/UserEntity.java | 4 +- .../dao/model/nosql/WidgetTypeEntity.java | 4 +- .../dao/model/nosql/WidgetsBundleEntity.java | 4 +- .../dao/model/sql/AdminSettingsEntity.java | 2 +- .../server/dao/model/sql/AlarmEntity.java | 2 +- .../server/dao/model/sql/AssetEntity.java | 2 +- .../server/dao/model/sql/AuditLogEntity.java | 6 +- .../model/sql/ComponentDescriptorEntity.java | 5 +- .../server/dao/model/sql/CustomerEntity.java | 6 +- .../server/dao/model/sql/DashboardEntity.java | 6 +- .../dao/model/sql/DashboardInfoEntity.java | 6 +- .../model/sql/DeviceCredentialsEntity.java | 6 +- .../server/dao/model/sql/DeviceEntity.java | 6 +- .../dao/model/sql/EntityViewEntity.java | 6 +- .../server/dao/model/sql/EventEntity.java | 6 +- .../server/dao/model/sql/RuleChainEntity.java | 6 +- .../server/dao/model/sql/RuleNodeEntity.java | 6 +- .../server/dao/model/sql/TenantEntity.java | 6 +- .../dao/model/sql/UserCredentialsEntity.java | 6 +- .../server/dao/model/sql/UserEntity.java | 6 +- .../dao/model/sql/WidgetTypeEntity.java | 6 +- .../dao/model/sql/WidgetsBundleEntity.java | 2 +- .../dao/nosql/CassandraAbstractModelDao.java | 6 +- .../server/dao/sql/JpaAbstractDao.java | 4 +- .../server/dao/sql/asset/JpaAssetDao.java | 8 +-- .../server/dao/sql/audit/JpaAuditLogDao.java | 9 +-- ...ctComponentDescriptorInsertRepository.java | 8 +-- ...qlComponentDescriptorInsertRepository.java | 2 +- .../JpaBaseComponentDescriptorDao.java | 4 +- .../dao/sql/customer/JpaCustomerDao.java | 2 +- .../sql/dashboard/JpaDashboardInfoDao.java | 2 +- .../server/dao/sql/device/JpaDeviceDao.java | 10 +-- .../dao/sql/entityview/JpaEntityViewDao.java | 8 +-- .../event/AbstractEventInsertRepository.java | 2 +- .../sql/event/HsqlEventInsertRepository.java | 2 +- .../server/dao/sql/event/JpaBaseEventDao.java | 13 ++-- .../dao/sql/relation/JpaRelationDao.java | 5 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 2 +- .../server/dao/sql/tenant/JpaTenantDao.java | 2 +- .../server/dao/sql/user/JpaUserDao.java | 4 +- .../dao/sql/widget/JpaWidgetsBundleDao.java | 6 +- ...stractChunkedAggregationTimeseriesDao.java | 4 +- .../timescale/TimescaleTimeseriesDao.java | 4 +- pom.xml | 60 +++++++++++++---- 85 files changed, 369 insertions(+), 242 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index b1517b9406..a90d371f2a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -72,11 +72,11 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } } - public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId) { + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId) { return getOrCreateActor(context, ruleChainId, eId -> ruleChainService.findRuleChainById(TenantId.SYS_TENANT_ID, eId)); } - public ActorRef getOrCreateActor(ActorContext context, RuleChainId ruleChainId, Function provider) { + public ActorRef getOrCreateActor(akka.actor.ActorContext context, RuleChainId ruleChainId, Function provider) { return actors.computeIfAbsent(ruleChainId, eId -> { RuleChain ruleChain = provider.apply(eId); return context.actorOf(Props.create(new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain)) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java b/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java index 84818bd3e4..a2c0d08ade 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/ContextAwareActor.java @@ -16,14 +16,14 @@ package org.thingsboard.server.actors.service; import akka.actor.Terminated; -import akka.actor.UntypedActor; +import akka.actor.UntypedAbstractActor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.TbActorMsg; -public abstract class ContextAwareActor extends UntypedActor { +public abstract class ContextAwareActor extends UntypedAbstractActor { protected final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 50e3580405..3e9f8c7853 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -114,9 +114,7 @@ public abstract class AbstractControllerTest { */ private static final long DEFAULT_TIMEOUT = -1L; - protected MediaType contentType = new MediaType(MediaType.APPLICATION_JSON.getType(), - MediaType.APPLICATION_JSON.getSubtype(), - Charset.forName("utf8")); + protected MediaType contentType = MediaType.APPLICATION_JSON; protected MockMvc mockMvc; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java index 96a36e1f65..a41dd2ee21 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/HsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") public @interface HsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index 20954b8ac2..bcf1222988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java index 8b642de026..dde0a9a176 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "cassandra") public @interface NoSqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java index ebd0ef7d28..7b10e0cf03 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra") public @interface NoSqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java index ef73ec8f90..a888926bd0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") public @interface PsqlDao { } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java index f2a8800032..a215a16b29 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") public @interface PsqlTsAnyDao { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java index 0eea3367e1..408f81a845 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "sql") public @interface SqlDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java index 9a43c530a2..9be3321988 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") public @interface SqlTsAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java index abba0e985b..0b665c5695 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql") public @interface SqlTsDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java index 20745d790c..0541a47879 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsDao.java @@ -17,6 +17,10 @@ package org.thingsboard.server.dao.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") public @interface TimescaleDBTsDao { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index a61bf560ee..c83fa02fab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -21,7 +21,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -30,10 +29,12 @@ import java.util.concurrent.TimeUnit; @Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; - private final Map> storage; + private final ConcurrentHashMap> storage; + private volatile boolean stopped; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); + stopped = false; } public static InMemoryStorage getInstance() { @@ -67,19 +68,20 @@ public final class InMemoryStorage { entities.add((T) other); } } + if (entities.size() > 0) { + storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).addAll(entities); + } return entities; } catch (InterruptedException e) { - log.warn("Queue was interrupted", e); - return Collections.emptyList(); + if (!stopped) { + log.warn("Queue was interrupted", e); + } } } return Collections.emptyList(); } - public void commit(String topic) { - //TODO: 2.5 Until someone calls commit you should not allow to poll new elements. - if (storage.containsKey(topic)) { -// storage.get(topic).remove(); - } + public void stop() { + stopped = true; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 8f226ff2e9..2b81839540 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -15,18 +15,26 @@ */ package org.thingsboard.server.queue.memory; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +@Slf4j public class InMemoryTbQueueConsumer implements TbQueueConsumer { private final InMemoryStorage storage = InMemoryStorage.getInstance(); + private volatile Set partitions; + private volatile boolean stopped; + private volatile boolean subscribed; public InMemoryTbQueueConsumer(String topic) { this.topic = topic; + stopped = false; } private final String topic; @@ -38,26 +46,44 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public void subscribe() { - + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = true; } @Override public void subscribe(Set partitions) { - + this.partitions = partitions; + subscribed = true; } @Override public void unsubscribe() { - + stopped = true; } @Override public List poll(long durationInMillis) { - return storage.get(topic, durationInMillis); + if (subscribed) { + List messages = partitions + .stream() + .map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis)) + .flatMap(List::stream) + .map(msg -> (T) msg).collect(Collectors.toList()); + if (messages.size() > 0) { + return messages; + } + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to sleep.", e); + } + } + } + return Collections.emptyList(); } @Override public void commit() { - storage.commit(topic); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index cb1ff939d5..cfcd788a16 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -39,7 +39,7 @@ public class InMemoryTbQueueProducer implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - boolean result = storage.put(tpi.getTopic(), msg); + boolean result = storage.put(tpi.getFullTopicName(), msg); if (result) { if (callback != null) { callback.onSuccess(null); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index baad96182f..fbdbeaab0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -18,19 +18,16 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -44,74 +41,79 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { + private final PartitionService partitionService; private final TbQueueCoreSettings coreSettings; + private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings notificationSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; - public InMemoryMonolithQueueFactory(TbQueueCoreSettings coreSettings, + public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings) { + this.partitionService = partitionService; this.coreSettings = coreSettings; + this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; - this.notificationSettings = notificationSettings; + this.transportNotificationSettings = transportNotificationSettings; } @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(notificationSettings.getNotificationsTopic()); + public TbQueueProducer> createTransportNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(transportNotificationSettings.getNotificationsTopic()); } @Override - public TbQueueProducer> createRuleEngineMsgProducer() { + public TbQueueProducer> createRuleEngineMsgProducer() { return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); + public TbQueueProducer> createTbCoreMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); + public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { + return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic()); } @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); + public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); } @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic() + ".notifications"); + public TbQueueConsumer> createToCoreMsgConsumer() { + return new InMemoryTbQueueConsumer<>(coreSettings.getTopic()); } @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic() + ".notifications"); + public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); } @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(coreSettings.getTopic() + ".notifications"); + public TbQueueConsumer> createTransportApiRequestConsumer() { + return new InMemoryTbQueueConsumer<>(transportApiSettings.getRequestsTopic()); } @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new InMemoryTbQueueConsumer<>(ruleEngineSettings.getTopic() + ".notifications"); + public TbQueueProducer> createTransportApiResponseProducer() { + return new InMemoryTbQueueProducer<>(transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index c779127e1f..29660f2461 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; @@ -29,10 +28,9 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -40,32 +38,32 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory { - - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings notificationSettings; + private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbServiceInfoProvider serviceInfoProvider; - public InMemoryTbTransportQueueFactory(TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings notificationSettings) { - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; + public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceInfoProvider serviceInfoProvider) { this.transportApiSettings = transportApiSettings; - this.notificationSettings = notificationSettings; + this.transportNotificationSettings = transportNotificationSettings; + this.serviceInfoProvider = serviceInfoProvider; } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - InMemoryTbQueueProducer> producer = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); - InMemoryTbQueueConsumer> consumer = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic()); + InMemoryTbQueueProducer> producerTemplate = + new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + + InMemoryTbQueueConsumer> consumerTemplate = + new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(topic -> Futures.immediateFuture(null)); - templateBuilder.requestTemplate(producer); - templateBuilder.responseTemplate(consumer); + templateBuilder.queueAdmin(topic -> { + }); + templateBuilder.requestTemplate(producerTemplate); + templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); @@ -74,16 +72,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new InMemoryTbQueueProducer<>(ruleEngineSettings.getTopic()); + return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); + return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new InMemoryTbQueueConsumer<>(notificationSettings.getNotificationsTopic()); + return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index e0ff9e1032..ff4a69e2e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -18,10 +18,13 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -125,4 +128,9 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 3bf6d4667c..5708e0738c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -28,6 +29,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponse import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -114,4 +117,9 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueProducer> createTransportApiResponseProducer() { return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 52d28cccd7..e2755938d9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -26,6 +27,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -97,4 +100,9 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java index 664d7b382f..68e398131b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/component/CassandraBaseComponentDescriptorDao.java @@ -151,12 +151,12 @@ public class CassandraBaseComponentDescriptorDao extends CassandraAbstractSearch } private Optional saveIfNotExist(TenantId tenantId, ComponentDescriptorEntity entity) { - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } ResultSet rs = executeRead(tenantId, QueryBuilder.insertInto(getColumnFamilyName()) - .value(ModelConstants.ID_PROPERTY, entity.getId()) + .value(ModelConstants.ID_PROPERTY, entity.getUuid()) .value(ModelConstants.COMPONENT_DESCRIPTOR_NAME_PROPERTY, entity.getName()) .value(ModelConstants.COMPONENT_DESCRIPTOR_CLASS_PROPERTY, entity.getClazz()) .value(ModelConstants.COMPONENT_DESCRIPTOR_TYPE_PROPERTY, entity.getType()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java index bdd0201aa5..f2660e512d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java @@ -184,11 +184,11 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao> saveAsync(TenantId tenantId, EventEntity entity, boolean ifNotExists, int ttl) { - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } Insert insert = QueryBuilder.insertInto(getColumnFamilyName()) - .value(ModelConstants.ID_PROPERTY, entity.getId()) + .value(ModelConstants.ID_PROPERTY, entity.getUuid()) .value(ModelConstants.EVENT_TENANT_ID_PROPERTY, entity.getTenantId()) .value(ModelConstants.EVENT_ENTITY_TYPE_PROPERTY, entity.getEntityType()) .value(ModelConstants.EVENT_ENTITY_ID_PROPERTY, entity.getEntityId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java index 386220df55..b5bda06a47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseEntity.java @@ -19,8 +19,8 @@ import java.util.UUID; public interface BaseEntity extends ToData { - UUID getId(); + UUID getUuid(); - void setId(UUID id); + void setUuid(UUID id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java index d75262e311..44fd70f977 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseSqlEntity.java @@ -35,14 +35,15 @@ public abstract class BaseSqlEntity implements BaseEntity { protected String id; @Override - public UUID getId() { + public UUID getUuid() { if (id == null) { return null; } return UUIDConverter.fromString(id); } - public void setId(UUID id) { + @Override + public void setUuid(UUID id) { this.id = UUIDConverter.fromTimeUUID(id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java index 93cc934a06..5d8f5d671f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AdminSettingsEntity.java @@ -61,11 +61,11 @@ public final class AdminSettingsEntity implements BaseEntity { this.jsonValue = adminSettings.getJsonValue(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java index 0cc8dfab72..d2e906121f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java @@ -141,11 +141,11 @@ public final class AlarmEntity implements BaseEntity { } } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java index 2554e0c3ec..57ec0ee511 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AssetEntity.java @@ -94,11 +94,11 @@ public final class AssetEntity implements SearchTextEntity { this.additionalInfo = asset.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java index 04e77aad8a..b406a6b341 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AuditLogEntity.java @@ -94,12 +94,12 @@ public class AuditLogEntity implements BaseEntity { private String actionFailureDetails; @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java index 9a6827aa05..1af3b8a8fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/ComponentDescriptorEntity.java @@ -98,12 +98,12 @@ public class ComponentDescriptorEntity implements SearchTextEntity { this.additionalInfo = customer.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java index 9fe999a0e5..a48305f630 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardEntity.java @@ -98,11 +98,11 @@ public final class DashboardEntity implements SearchTextEntity { this.configuration = dashboard.getConfiguration(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java index b54252fdfc..69eb43744c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DashboardInfoEntity.java @@ -91,11 +91,11 @@ public class DashboardInfoEntity implements SearchTextEntity { } } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java index 6900014d21..316386f7c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/DeviceCredentialsEntity.java @@ -74,11 +74,11 @@ public final class DeviceCredentialsEntity implements BaseEntity { this.additionalInfo = device.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java index 80bc9a3276..5cf13665b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java @@ -161,4 +161,14 @@ public class EntityViewEntity implements SearchTextEntity { entityView.setAdditionalInfo(additionalInfo); return entityView; } + + @Override + public UUID getUuid() { + return getId(); + } + + @Override + public void setUuid(UUID id) { + this.id = id; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java index 2645d29df3..8c753a743a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/EventEntity.java @@ -94,12 +94,12 @@ public class EventEntity implements BaseEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java index bb6a194801..7021ff708e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleChainEntity.java @@ -102,12 +102,12 @@ public class RuleChainEntity implements SearchTextEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java index e2dc2fa244..9a086fe568 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/RuleNodeEntity.java @@ -96,12 +96,12 @@ public class RuleNodeEntity implements SearchTextEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java index 0ce7990755..29a51e2539 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java @@ -110,11 +110,11 @@ public final class TenantEntity implements SearchTextEntity { this.additionalInfo = tenant.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java index 50b54bd8c9..6854d61f39 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserCredentialsEntity.java @@ -75,11 +75,11 @@ public final class UserCredentialsEntity implements BaseEntity this.resetToken = userCredentials.getResetToken(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java index d8399fc2b5..0c7625da78 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/UserEntity.java @@ -101,11 +101,11 @@ public final class UserEntity implements SearchTextEntity { this.additionalInfo = user.getAdditionalInfo(); } - public UUID getId() { + public UUID getUuid() { return id; } - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java index 65bc19f421..24ac3980cf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetTypeEntity.java @@ -82,12 +82,12 @@ public final class WidgetTypeEntity implements BaseEntity { } @Override - public UUID getId() { + public UUID getUuid() { return id; } @Override - public void setId(UUID id) { + public void setUuid(UUID id) { this.id = id; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java index b70a0f095b..8c11c05685 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/WidgetsBundleEntity.java @@ -82,12 +82,12 @@ public final class WidgetsBundleEntity implements SearchTextEntity impl public AdminSettingsEntity(AdminSettings adminSettings) { if (adminSettings.getId() != null) { - this.setId(adminSettings.getId().getId()); + this.setUuid(adminSettings.getId().getId()); } this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index 14adcdcd9c..fb4efd25d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -114,7 +114,7 @@ public final class AlarmEntity extends BaseSqlEntity implements BaseEntit public AlarmEntity(Alarm alarm) { if (alarm.getId() != null) { - this.setId(alarm.getId().getId()); + this.setUuid(alarm.getId().getId()); } if (alarm.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(alarm.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java index 0471d4a97d..6eff4cd521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AssetEntity.java @@ -78,7 +78,7 @@ public final class AssetEntity extends BaseSqlEntity implements SearchTex public AssetEntity(Asset asset) { if (asset.getId() != null) { - this.setId(asset.getId().getId()); + this.setUuid(asset.getId().getId()); } if (asset.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(asset.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java index 4de738ac73..e2573ea0d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AuditLogEntity.java @@ -103,7 +103,7 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit public AuditLogEntity(AuditLog auditLog) { if (auditLog.getId() != null) { - this.setId(auditLog.getId().getId()); + this.setUuid(auditLog.getId().getId()); } if (auditLog.getTenantId() != null) { this.tenantId = toString(auditLog.getTenantId().getId()); @@ -128,8 +128,8 @@ public class AuditLogEntity extends BaseSqlEntity implements BaseEntit @Override public AuditLog toData() { - AuditLog auditLog = new AuditLog(new AuditLogId(getId())); - auditLog.setCreatedTime(UUIDs.unixTimestamp(getId())); + AuditLog auditLog = new AuditLog(new AuditLogId(this.getUuid())); + auditLog.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { auditLog.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java index 26946cf207..d5b287fe9b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java @@ -34,7 +34,6 @@ import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; -import javax.persistence.UniqueConstraint; @Data @EqualsAndHashCode(callSuper = true) @@ -72,7 +71,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity implements Sea public CustomerEntity(Customer customer) { if (customer.getId() != null) { - this.setId(customer.getId().getId()); + this.setUuid(customer.getId().getId()); } this.tenantId = UUIDConverter.fromTimeUUID(customer.getTenantId().getId()); this.title = customer.getTitle(); @@ -111,8 +111,8 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Override public Customer toData() { - Customer customer = new Customer(new CustomerId(getId())); - customer.setCreatedTime(UUIDs.unixTimestamp(getId())); + Customer customer = new Customer(new CustomerId(this.getUuid())); + customer.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); customer.setTenantId(new TenantId(UUIDConverter.fromString(tenantId))); customer.setTitle(title); customer.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index b606c35d4e..07cec4d12f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -75,7 +75,7 @@ public final class DashboardEntity extends BaseSqlEntity implements S public DashboardEntity(Dashboard dashboard) { if (dashboard.getId() != null) { - this.setId(dashboard.getId().getId()); + this.setUuid(dashboard.getId().getId()); } if (dashboard.getTenantId() != null) { this.tenantId = toString(dashboard.getTenantId().getId()); @@ -103,8 +103,8 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Override public Dashboard toData() { - Dashboard dashboard = new Dashboard(new DashboardId(this.getId())); - dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getId())); + Dashboard dashboard = new Dashboard(new DashboardId(this.getUuid())); + dashboard.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboard.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java index b1ecb2572b..f26cf21934 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardInfoEntity.java @@ -66,7 +66,7 @@ public class DashboardInfoEntity extends BaseSqlEntity implements public DashboardInfoEntity(DashboardInfo dashboardInfo) { if (dashboardInfo.getId() != null) { - this.setId(dashboardInfo.getId().getId()); + this.setUuid(dashboardInfo.getId().getId()); } if (dashboardInfo.getTenantId() != null) { this.tenantId = toString(dashboardInfo.getTenantId().getId()); @@ -97,8 +97,8 @@ public class DashboardInfoEntity extends BaseSqlEntity implements @Override public DashboardInfo toData() { - DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(getId())); - dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(getId())); + DashboardInfo dashboardInfo = new DashboardInfo(new DashboardId(this.getUuid())); + dashboardInfo.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { dashboardInfo.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java index 1d47c7d63c..ba9d7ce637 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceCredentialsEntity.java @@ -57,7 +57,7 @@ public final class DeviceCredentialsEntity extends BaseSqlEntity implements SearchT public DeviceEntity(Device device) { if (device.getId() != null) { - this.setId(device.getId().getId()); + this.setUuid(device.getId().getId()); } if (device.getTenantId() != null) { this.tenantId = toString(device.getTenantId().getId()); @@ -95,8 +95,8 @@ public final class DeviceEntity extends BaseSqlEntity implements SearchT @Override public Device toData() { - Device device = new Device(new DeviceId(getId())); - device.setCreatedTime(UUIDs.unixTimestamp(getId())); + Device device = new Device(new DeviceId(this.getUuid())); + device.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { device.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java index 6a755d2803..fd446cc6ef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityViewEntity.java @@ -99,7 +99,7 @@ public class EntityViewEntity extends BaseSqlEntity implements Searc public EntityViewEntity(EntityView entityView) { if (entityView.getId() != null) { - this.setId(entityView.getId().getId()); + this.setUuid(entityView.getId().getId()); } if (entityView.getEntityId() != null) { this.entityId = toString(entityView.getEntityId().getId()); @@ -136,8 +136,8 @@ public class EntityViewEntity extends BaseSqlEntity implements Searc @Override public EntityView toData() { - EntityView entityView = new EntityView(new EntityViewId(getId())); - entityView.setCreatedTime(UUIDs.unixTimestamp(getId())); + EntityView entityView = new EntityView(new EntityViewId(this.getUuid())); + entityView.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (entityId != null) { entityView.setEntityId(EntityIdFactory.getByTypeAndId(entityType.name(), toUUID(entityId).toString())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..6cade0f576 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -75,7 +75,7 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity implements SearchT public RuleChainEntity(RuleChain ruleChain) { if (ruleChain.getId() != null) { - this.setId(ruleChain.getUuidId()); + this.setUuid(ruleChain.getUuidId()); } this.tenantId = toString(DaoUtil.getId(ruleChain.getTenantId())); this.name = ruleChain.getName(); @@ -100,8 +100,8 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Override public RuleChain toData() { - RuleChain ruleChain = new RuleChain(new RuleChainId(getId())); - ruleChain.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleChain ruleChain = new RuleChain(new RuleChainId(this.getUuid())); + ruleChain.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); ruleChain.setTenantId(new TenantId(toUUID(tenantId))); ruleChain.setName(name); if (firstRuleNodeId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index f36edc260b..1a7866418a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -69,7 +69,7 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex public RuleNodeEntity(RuleNode ruleNode) { if (ruleNode.getId() != null) { - this.setId(ruleNode.getUuidId()); + this.setUuid(ruleNode.getUuidId()); } if (ruleNode.getRuleChainId() != null) { this.ruleChainId = toString(DaoUtil.getId(ruleNode.getRuleChainId())); @@ -94,8 +94,8 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Override public RuleNode toData() { - RuleNode ruleNode = new RuleNode(new RuleNodeId(getId())); - ruleNode.setCreatedTime(UUIDs.unixTimestamp(getId())); + RuleNode ruleNode = new RuleNode(new RuleNodeId(this.getUuid())); + ruleNode.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (ruleChainId != null) { ruleNode.setRuleChainId(new RuleChainId(toUUID(ruleChainId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 239d3698b4..10b00f38ea 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -82,7 +82,7 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT public TenantEntity(Tenant tenant) { if (tenant.getId() != null) { - this.setId(tenant.getId().getId()); + this.setUuid(tenant.getId().getId()); } this.title = tenant.getTitle(); this.region = tenant.getRegion(); @@ -113,8 +113,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Override public Tenant toData() { - Tenant tenant = new Tenant(new TenantId(getId())); - tenant.setCreatedTime(UUIDs.unixTimestamp(getId())); + Tenant tenant = new Tenant(new TenantId(this.getUuid())); + tenant.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); tenant.setTitle(title); tenant.setRegion(region); tenant.setCountry(country); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java index 70593ca610..6957f87961 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserCredentialsEntity.java @@ -56,7 +56,7 @@ public final class UserCredentialsEntity extends BaseSqlEntity public UserCredentialsEntity(UserCredentials userCredentials) { if (userCredentials.getId() != null) { - this.setId(userCredentials.getId().getId()); + this.setUuid(userCredentials.getId().getId()); } if (userCredentials.getUserId() != null) { this.userId = toString(userCredentials.getUserId().getId()); @@ -69,8 +69,8 @@ public final class UserCredentialsEntity extends BaseSqlEntity @Override public UserCredentials toData() { - UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(getId())); - userCredentials.setCreatedTime(UUIDs.unixTimestamp(getId())); + UserCredentials userCredentials = new UserCredentials(new UserCredentialsId(this.getUuid())); + userCredentials.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (userId != null) { userCredentials.setUserId(new UserId(toUUID(userId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java index 0100110daa..3af671583e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/UserEntity.java @@ -81,7 +81,7 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< public UserEntity(User user) { if (user.getId() != null) { - this.setId(user.getId().getId()); + this.setUuid(user.getId().getId()); } this.authority = user.getAuthority(); if (user.getTenantId() != null) { @@ -108,8 +108,8 @@ public class UserEntity extends BaseSqlEntity implements SearchTextEntity< @Override public User toData() { - User user = new User(new UserId(getId())); - user.setCreatedTime(UUIDs.unixTimestamp(getId())); + User user = new User(new UserId(this.getUuid())); + user.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); user.setAuthority(authority); if (tenantId != null) { user.setTenantId(new TenantId(fromString(tenantId))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 8e81b30e8f..0cb69b9d8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -62,7 +62,7 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement public WidgetTypeEntity(WidgetType widgetType) { if (widgetType.getId() != null) { - this.setId(widgetType.getId().getId()); + this.setUuid(widgetType.getId().getId()); } if (widgetType.getTenantId() != null) { this.tenantId = toString(widgetType.getTenantId().getId()); @@ -75,8 +75,8 @@ public final class WidgetTypeEntity extends BaseSqlEntity implement @Override public WidgetType toData() { - WidgetType widgetType = new WidgetType(new WidgetTypeId(getId())); - widgetType.setCreatedTime(UUIDs.unixTimestamp(getId())); + WidgetType widgetType = new WidgetType(new WidgetTypeId(this.getUuid())); + widgetType.setCreatedTime(UUIDs.unixTimestamp(this.getUuid())); if (tenantId != null) { widgetType.setTenantId(new TenantId(toUUID(tenantId))); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 7df6ea9fe7..75ea383090 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -55,7 +55,7 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl public WidgetsBundleEntity(WidgetsBundle widgetsBundle) { if (widgetsBundle.getId() != null) { - this.setId(widgetsBundle.getId().getId()); + this.setUuid(widgetsBundle.getId().getId()); } if (widgetsBundle.getTenantId() != null) { this.tenantId = UUIDConverter.fromTimeUUID(widgetsBundle.getTenantId().getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java index dd56136e76..2da9915bb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java @@ -132,10 +132,10 @@ public abstract class CassandraAbstractModelDao, D> exte protected EntityResultSet saveWithResult(TenantId tenantId, E entity) { log.debug("Save entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } else if (isDeleteOnSave()) { - removeById(tenantId, entity.getId()); + removeById(tenantId, entity.getUuid()); } Statement saveStatement = getSaveQuery(entity); saveStatement.setConsistencyLevel(cluster.getDefaultWriteConsistencyLevel()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index d3ce07f901..edc9c97a9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -58,8 +58,8 @@ public abstract class JpaAbstractDao, D> } setSearchText(entity); log.debug("Saving entity {}", entity); - if (entity.getId() == null) { - entity.setId(UUIDs.timeBased()); + if (entity.getUuid() == null) { + entity.setUuid(UUIDs.timeBased()); } entity = getCrudRepository().save(entity); return DaoUtil.getData(entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index aaddc15eb8..c4482a3247 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -69,7 +69,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im fromTimeUUID(tenantId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -86,7 +86,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im fromTimeUUID(customerId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -109,7 +109,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -121,7 +121,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index c0a260e9da..566d9c3225 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -16,8 +16,6 @@ package org.thingsboard.server.dao.sql.audit; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -39,14 +37,11 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTimeDao; import org.thingsboard.server.dao.util.SqlDao; -import javax.annotation.PreDestroy; import javax.persistence.criteria.Predicate; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.concurrent.Executors; -import static org.springframework.data.jpa.domain.Specifications.where; import static org.thingsboard.server.dao.model.ModelConstants.ID_PROPERTY; @Component @@ -118,8 +113,8 @@ public class JpaAuditLogDao extends JpaAbstractDao imp Specification timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "id"); Specification fieldsSpec = getEntityFieldsSpec(tenantId, entityId, customerId, userId, actionTypes); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); - return DaoUtil.convertDataList(auditLogRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); + return DaoUtil.convertDataList(auditLogRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); } private Specification getEntityFieldsSpec(UUID tenantId, EntityId entityId, CustomerId customerId, UserId userId, List actionTypes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java index c634e5a064..d8272e3a57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java @@ -48,17 +48,17 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com } catch (Throwable throwable) { transactionManager.rollback(insertTransaction); if (throwable.getCause() instanceof ConstraintViolationException) { - log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getId(), entity.getName(), entity.getType()); + log.trace("Insert request leaded in a violation of a defined integrity constraint {} for Component Descriptor with id {}, name {} and entityType {}", throwable.getMessage(), entity.getUuid(), entity.getName(), entity.getType()); TransactionStatus transaction = getTransactionStatus(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { componentDescriptorEntity = processSaveOrUpdate(entity, insertOrUpdateOnUniqueKeyConflict); } catch (Throwable th) { - log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the update statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); transactionManager.rollback(transaction); } transactionManager.commit(transaction); } else { - log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getId(), entity.getName(), entity.getType()); + log.trace("Could not execute the insert statement for Component Descriptor with id {}, name {} and entityType {}", entity.getUuid(), entity.getName(), entity.getType()); } } return componentDescriptorEntity; @@ -69,7 +69,7 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com protected Query getQuery(ComponentDescriptorEntity entity, String query) { return entityManager.createNativeQuery(query, ComponentDescriptorEntity.class) - .setParameter("id", UUIDConverter.fromTimeUUID(entity.getId())) + .setParameter("id", UUIDConverter.fromTimeUUID(entity.getUuid())) .setParameter("actions", entity.getActions()) .setParameter("clazz", entity.getClazz()) .setParameter("configuration_descriptor", entity.getConfigurationDescriptor().toString()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index e281bdeec4..c33dd4f5e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -40,7 +40,7 @@ public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDe @Override protected ComponentDescriptorEntity doProcessSaveOrUpdate(ComponentDescriptorEntity entity, String query) { getQuery(entity, query).executeUpdate(); - return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getId())); + return entityManager.find(ComponentDescriptorEntity.class, UUIDConverter.fromTimeUUID(entity.getUuid())); } private static String getInsertString(String conflictStatement) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index 938b030d53..e66937166f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -94,7 +94,7 @@ public class JpaBaseComponentDescriptorDao extends JpaAbstractSearchTextDao deviceRepository.findByTenantId( fromTimeUUID(tenantId), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } else { return DaoUtil.convertDataList( deviceRepository.findByTenantId( fromTimeUUID(tenantId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } @@ -95,7 +95,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao fromTimeUUID(customerId), Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -118,7 +118,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -130,7 +130,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao type, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 91a9181908..385a901dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -70,7 +70,7 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "id"); Specification fieldsSpec = getEntityFieldsSpec(tenantId, entityId, eventType); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); - return DaoUtil.convertDataList(eventRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, ID_PROPERTY); + return DaoUtil.convertDataList(eventRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent()); } @Override @@ -130,7 +129,7 @@ public class JpaBaseEventDao extends JpaAbstractSearchTimeDao timeSearchSpec = JpaAbstractSearchTimeDao.getTimeSearchPageSpec(pageLink, "toId"); Specification fieldsSpec = getEntityFieldsSpec(from, relationType, typeGroup, childType); Sort.Direction sortDirection = pageLink.isAscOrder() ? Sort.Direction.ASC : Sort.Direction.DESC; - Pageable pageable = new PageRequest(0, pageLink.getLimit(), sortDirection, "toId"); + Pageable pageable = PageRequest.of(0, pageLink.getLimit(), sortDirection, "toId"); return service.submit(() -> - DaoUtil.convertDataList(relationRepository.findAll(where(timeSearchSpec).and(fieldsSpec), pageable).getContent())); + DaoUtil.convertDataList(relationRepository.findAll(Specification.where(timeSearchSpec).and(fieldsSpec), pageable).getContent())); } private Specification getEntityFieldsSpec(EntityId from, String relationType, RelationTypeGroup typeGroup, EntityType childType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 415cbd15c1..b1e4745223 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -60,7 +60,7 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao region, Objects.toString(pageLink.getTextSearch(), ""), pageLink.getIdOffset() == null ? NULL_UUID_STR : UUIDConverter.fromTimeUUID(pageLink.getIdOffset()), - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 33b7c3c1e9..f30f6ba6d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -71,7 +71,7 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), Objects.toString(pageLink.getTextSearch(), ""), Authority.TENANT_ADMIN, - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } @Override @@ -84,7 +84,7 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple pageLink.getIdOffset() == null ? NULL_UUID_STR : fromTimeUUID(pageLink.getIdOffset()), Objects.toString(pageLink.getTextSearch(), ""), Authority.CUSTOMER_USER, - new PageRequest(0, pageLink.getLimit()))); + PageRequest.of(0, pageLink.getLimit()))); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 228c596642..af6785e69e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -68,7 +68,7 @@ public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao tsKvEntity.setStrKey(query.getKey())); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index bf4cf5d9e6..0a2e210648 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -109,8 +109,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( + PageRequest.of(0, query.getLimit(), + Sort.by(Sort.Direction.fromString( query.getOrderBy()), "ts"))); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); diff --git a/pom.xml b/pom.xml index 383fbf0edc..8876e68538 100755 --- a/pom.xml +++ b/pom.xml @@ -30,11 +30,11 @@ ${basedir} thingsboard - 2.1.3.RELEASE - 5.1.5.RELEASE - 5.1.4.RELEASE - 2.1.5.RELEASE - 2.9.0 + 2.2.4.RELEASE + 5.2.2.RELEASE + 5.2.2.RELEASE + 2.2.4.RELEASE + 3.1.0 0.7.0 2.2.0 4.12 @@ -51,12 +51,12 @@ 1.6 2.5 1.4 - 2.9.9.3 - 2.9.9 - 2.9.9 + 2.10.2 + 2.10.2 + 2.10.2 2.2.6 - 2.11 - 2.4.2 + 2.13 + 2.6.3 1.0.2 2.6.2 1.7 @@ -68,7 +68,7 @@ 1.22.1 1.16.18 1.1.0 - 4.1.37.Final + 4.1.45.Final 1.5.0 4.8.0 2.19.1 @@ -77,7 +77,7 @@ 1.0.0 0.7 1.15.0 - 1.56 + 1.64 2.0.1 2.5.0 2.5.3 @@ -92,12 +92,15 @@ 4.1.1 2.57 2.7.7 - 1.23 + 1.25 + 1.3.10 1.11.747 1.84.0 3.2.0 1.5.0 1.4.3 + 1.9.4 + 3.2.2 @@ -915,6 +918,37 @@ uap-java ${ua-parser.version} + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + commons-collections + commons-collections + ${commons-collections.version} + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + org.apache.struts + struts-core + ${struts.version} + + + org.apache.struts + struts-taglib + ${struts.version} + + + org.apache.struts + struts-tiles + ${struts.version} + + From d074ee7e23b60b95de5f2ca987958680adcd481e Mon Sep 17 00:00:00 2001 From: Maksym Dudnik Date: Fri, 10 Apr 2020 13:35:50 +0300 Subject: [PATCH 058/196] Rule node Updated Translation --- .../public/static/rulenode/rulenode-core-config.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js index db0d0cdbb0..fd5d9b7c62 100644 --- a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js +++ b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js @@ -1,6 +1,6 @@ !function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(103)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; -},function(e,t){e.exports="
tb.rulenode.interval-seconds-required
tb.rulenode.min-interval-seconds-message
tb.rulenode.output-timeseries-key-prefix-required
"},function(e,t){e.exports='
{{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
tb.rulenode.period-seconds-required
tb.rulenode.min-period-0-seconds-message
tb.rulenode.period-in-seconds-pattern-required
tb.rulenode.period-in-seconds-pattern-hint
tb.rulenode.max-pending-messages-required
tb.rulenode.max-pending-messages-range
tb.rulenode.max-pending-messages-range
'},function(e,t){e.exports="
tb.rulenode.gcp-project-id-required
tb.rulenode.pubsub-topic-name-required
{{ 'action.remove' | translate }} close
tb.rulenode.no-file
{{configuration.serviceAccountKeyFileName}}
tb.rulenode.message-attributes-hint
"},function(e,t){e.exports='
{{ property }}
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
{{ \'tb.rulenode.automatic-recovery\' | translate }}
tb.rulenode.min-connection-timeout-ms-message
tb.rulenode.min-handshake-timeout-ms-message
'},function(e,t){e.exports='
tb.rulenode.endpoint-url-pattern-required
tb.rulenode.endpoint-url-pattern-hint
 {{ type }} {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}
tb.rulenode.read-timeout-hint
tb.rulenode.max-parallel-requests-count-hint
tb.rulenode.headers-hint
{{ \'tb.rulenode.use-redis-queue\' | translate }}
{{ \'tb.rulenode.trim-redis-queue\' | translate }}
'},function(e,t){e.exports="
"},function(e,t){e.exports="
tb.rulenode.timeout-required
tb.rulenode.min-timeout-message
"},function(e,t){e.exports='
tb.rulenode.custom-table-name-required
tb.rulenode.custom-table-hint
'},function(e,t){e.exports='
{{ \'tb.rulenode.use-system-smtp-settings\' | translate }}
{{smtpProtocol.toUpperCase()}}
tb.rulenode.smtp-host-required
tb.rulenode.smtp-port-required
tb.rulenode.smtp-port-range
tb.rulenode.smtp-port-range
tb.rulenode.timeout-required
tb.rulenode.min-timeout-msec-message
 {{ \'tb.rulenode.enable-tls\' | translate }} {{tlsVersion}}
'},function(e,t){e.exports="
tb.rulenode.topic-arn-pattern-required
tb.rulenode.topic-arn-pattern-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
"},function(e,t){e.exports='
{{ type.name | translate }}
tb.rulenode.queue-url-pattern-required
tb.rulenode.queue-url-pattern-hint
tb.rulenode.min-delay-seconds-message
tb.rulenode.max-delay-seconds-message
tb.rulenode.message-attributes-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
'},function(e,t){e.exports="
tb.rulenode.default-ttl-required
tb.rulenode.min-default-ttl-message
"},function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'relation.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-type
device.device-types
"},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
{{\'tb.rulenode.entity-details-\'+item.toLowerCase() | translate}} tb.rulenode.no-entity-details-matching {{\'tb.rulenode.entity-details-\'+$chip.toLowerCase() | translate}} {{ \'tb.rulenode.add-to-metadata\' | translate }}
tb.rulenode.add-to-metadata-hint
'},function(e,t){e.exports='
{{ type }}
tb.rulenode.fetch-mode-hint
{{ type }}
tb.rulenode.order-by-hint
tb.rulenode.limit-hint
{{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}
tb.rulenode.use-metadata-interval-patterns-hint
tb.rulenode.start-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.end-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.start-interval-pattern-required
tb.rulenode.start-interval-pattern-hint
tb.rulenode.end-interval-pattern-required
tb.rulenode.end-interval-pattern-hint
'; -},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},31,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'relation.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(75),r=a(i),o=n(53),l=a(o),s=n(58),d=a(s),u=n(55),c=a(u),m=n(54),g=a(m),p=n(62),f=a(p),b=n(69),v=a(b),y=n(70),h=a(y),q=n(68),k=a(q),x=n(61),$=a(x),T=n(73),C=a(T),w=n(74),M=a(w),S=n(67),N=a(S),_=n(63),F=a(_),E=n(72),P=a(E),A=n(65),V=a(A),I=n(64),O=a(I),j=n(52),D=a(j),L=n(76),R=a(L),K=n(57),U=a(K),z=n(56),H=a(z),B=n(71),G=a(B),Y=n(59),Q=a(Y),W=n(66),J=a(W);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",N.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",O.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader; -t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1.0","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(84),r=a(i),o=n(85),l=a(o),s=n(80),d=a(s),u=n(86),c=a(u),m=n(79),g=a(m),p=n(87),f=a(p),b=n(82),v=a(b),y=n(81),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(95),r=a(i),o=n(93),l=a(o),s=n(96),d=a(s),u=n(90),c=a(u),m=n(94),g=a(m),p=n(89),f=a(p),b=n(91),v=a(b),y=n(88),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(47),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(99),r=a(i),o=n(101),l=a(o),s=n(102),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(106),r=a(i),o=n(92),l=a(o),s=n(83),d=a(s),u=n(100),c=a(u),m=n(60),g=a(m),p=n(78),f=a(p),b=n(98),v=a(b),y=n(77),h=a(y),q=n(97),k=a(q),x=n(105),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata", +},function(e,t){e.exports="
tb.rulenode.interval-seconds-required
tb.rulenode.min-interval-seconds-message
tb.rulenode.output-timeseries-key-prefix-required
"},function(e,t){e.exports='
{{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
tb.rulenode.period-seconds-required
tb.rulenode.min-period-0-seconds-message
tb.rulenode.period-in-seconds-pattern-required
tb.rulenode.period-in-seconds-pattern-hint
tb.rulenode.max-pending-messages-required
tb.rulenode.max-pending-messages-range
tb.rulenode.max-pending-messages-range
'},function(e,t){e.exports="
tb.rulenode.gcp-project-id-required
tb.rulenode.pubsub-topic-name-required
{{ 'action.remove' | translate }} close
tb.rulenode.no-file
{{configuration.serviceAccountKeyFileName}}
tb.rulenode.message-attributes-hint
"},function(e,t){e.exports='
{{ property }}
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
{{ \'tb.rulenode.automatic-recovery\' | translate }}
tb.rulenode.min-connection-timeout-ms-message
tb.rulenode.min-handshake-timeout-ms-message
'},function(e,t){e.exports='
tb.rulenode.endpoint-url-pattern-required
tb.rulenode.endpoint-url-pattern-hint
 {{ type }} {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}
tb.rulenode.read-timeout-hint
tb.rulenode.max-parallel-requests-count-hint
tb.rulenode.headers-hint
{{ \'tb.rulenode.use-redis-queue\' | translate }}
{{ \'tb.rulenode.trim-redis-queue\' | translate }}
'},function(e,t){e.exports="
"},function(e,t){e.exports="
tb.rulenode.timeout-required
tb.rulenode.min-timeout-message
"},function(e,t){e.exports='
tb.rulenode.custom-table-name-required
tb.rulenode.custom-table-hint
'},function(e,t){e.exports='
{{ \'tb.rulenode.use-system-smtp-settings\' | translate }}
{{smtpProtocol.toUpperCase()}}
tb.rulenode.smtp-host-required
tb.rulenode.smtp-port-required
tb.rulenode.smtp-port-range
tb.rulenode.smtp-port-range
tb.rulenode.timeout-required
tb.rulenode.min-timeout-msec-message
 {{ \'tb.rulenode.enable-tls\' | translate }} {{tlsVersion}}
'},function(e,t){e.exports="
tb.rulenode.topic-arn-pattern-required
tb.rulenode.topic-arn-pattern-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
"},function(e,t){e.exports='
{{ type.name | translate }}
tb.rulenode.queue-url-pattern-required
tb.rulenode.queue-url-pattern-hint
tb.rulenode.min-delay-seconds-message
tb.rulenode.max-delay-seconds-message
tb.rulenode.message-attributes-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
'},function(e,t){e.exports="
tb.rulenode.default-ttl-required
tb.rulenode.min-default-ttl-message
"},function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-type
device.device-types
"},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
{{\'tb.rulenode.entity-details-\'+item.toLowerCase() | translate}} tb.rulenode.no-entity-details-matching {{\'tb.rulenode.entity-details-\'+$chip.toLowerCase() | translate}} {{ \'tb.rulenode.add-to-metadata\' | translate }}
tb.rulenode.add-to-metadata-hint
'},function(e,t){e.exports='
{{ type }}
tb.rulenode.fetch-mode-hint
{{ type }}
tb.rulenode.order-by-hint
tb.rulenode.limit-hint
{{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}
tb.rulenode.use-metadata-interval-patterns-hint
tb.rulenode.start-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.end-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.start-interval-pattern-required
tb.rulenode.start-interval-pattern-hint
tb.rulenode.end-interval-pattern-required
tb.rulenode.end-interval-pattern-hint
'; +},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},31,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(75),r=a(i),o=n(53),l=a(o),s=n(58),d=a(s),u=n(55),c=a(u),m=n(54),g=a(m),p=n(62),f=a(p),b=n(69),v=a(b),y=n(70),h=a(y),q=n(68),k=a(q),x=n(61),$=a(x),T=n(73),C=a(T),w=n(74),M=a(w),S=n(67),N=a(S),_=n(63),F=a(_),E=n(72),P=a(E),A=n(65),V=a(A),I=n(64),O=a(I),j=n(52),D=a(j),L=n(76),R=a(L),K=n(57),U=a(K),z=n(56),H=a(z),B=n(71),G=a(B),Y=n(59),Q=a(Y),W=n(66),J=a(W);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",N.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",O.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader; +t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(84),r=a(i),o=n(85),l=a(o),s=n(80),d=a(s),u=n(86),c=a(u),m=n(79),g=a(m),p=n(87),f=a(p),b=n(82),v=a(b),y=n(81),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(95),r=a(i),o=n(93),l=a(o),s=n(96),d=a(s),u=n(90),c=a(u),m=n(94),g=a(m),p=n(89),f=a(p),b=n(91),v=a(b),y=n(88),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(47),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(99),r=a(i),o=n(101),l=a(o),s=n(102),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(106),r=a(i),o=n(92),l=a(o),s=n(83),d=a(s),u=n(100),c=a(u),m=n(60),g=a(m),p=n(78),f=a(p),b=n(98),v=a(b),y=n(77),h=a(y),q=n(97),k=a(q),x=n(105),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata", header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16"},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(104),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); //# sourceMappingURL=rulenode-core-config.js.map \ No newline at end of file From f4ce41be20d45e3be95337f62d50301d5ef700e4 Mon Sep 17 00:00:00 2001 From: nordmif Date: Thu, 9 Apr 2020 18:41:43 +0300 Subject: [PATCH 059/196] added logging for failure to send mail --- .../org/thingsboard/server/service/mail/DefaultMailService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java b/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java index baa0d417a3..457d497e57 100644 --- a/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java +++ b/application/src/main/java/org/thingsboard/server/service/mail/DefaultMailService.java @@ -277,6 +277,7 @@ public class DefaultMailService implements MailService { } else { message = exception.getMessage(); } + log.warn("Unable to send mail: {}", message); return new ThingsboardException(String.format("Unable to send mail: %s", message), ThingsboardErrorCode.GENERAL); } From 7869fb5ef48eb2892aa5c0818c85017b0d86d08a Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 13 Apr 2020 10:24:40 +0300 Subject: [PATCH 060/196] added to tenant fields isolatedTbCore and isolatedTbRuleEngine (#2611) Co-authored-by: Andrew Shvayka --- .../CassandraDatabaseUpgradeService.java | 14 ++++ .../install/SqlDatabaseUpgradeService.java | 1 + .../server/common/data/Tenant.java | 22 +++++++ .../server/dao/model/ModelConstants.java | 2 + .../server/dao/model/nosql/TenantEntity.java | 35 ++++++++-- .../server/dao/model/sql/TenantEntity.java | 10 +++ .../resources/cassandra/schema-entities.cql | 2 + .../resources/sql/schema-entities-hsql.sql | 4 +- .../main/resources/sql/schema-entities.sql | 4 +- ui/src/app/locale/locale.constant-en_US.json | 6 +- ui/src/app/tenant/tenant-fieldset.tpl.html | 64 ++++++++++++------- 11 files changed, 134 insertions(+), 30 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java index 721d43bf9f..573d152635 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java @@ -290,6 +290,20 @@ public class CassandraDatabaseUpgradeService extends AbstractCassandraDatabaseUp log.info("Attributes updated."); } catch (InvalidQueryException e) { } + + String updateTenantCoreTableStmt = "alter table tenant add isolated_tb_core boolean"; + String updateTenantRuleEngineTableStmt = "alter table tenant add isolated_tb_rule_engine boolean"; + + try { + log.info("Updating tenant..."); + cluster.getSession().execute(updateTenantCoreTableStmt); + Thread.sleep(2500); + + cluster.getSession().execute(updateTenantRuleEngineTableStmt); + Thread.sleep(2500); + log.info("Tenant updated."); + } catch (InvalidQueryException e) { + } log.info("Schema updated."); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..bcbd77ef28 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); log.info("Schema updated."); } break; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index b1f6a967b6..766d405e25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -29,6 +29,8 @@ public class Tenant extends ContactBased implements HasTenantId { private String title; private String region; + private boolean isolatedTbCore; + private boolean isolatedTbRuleEngine; public Tenant() { super(); @@ -72,6 +74,22 @@ public class Tenant extends ContactBased implements HasTenantId { this.region = region; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchText() { return getTitle(); @@ -84,6 +102,10 @@ public class Tenant extends ContactBased implements HasTenantId { builder.append(title); builder.append(", region="); builder.append(region); + builder.append(", isolatedTbCore="); + builder.append(isolatedTbCore); + builder.append(", isolatedTbRuleEngine="); + builder.append(isolatedTbRuleEngine); builder.append(", additionalInfo="); builder.append(getAdditionalInfo()); builder.append(", country="); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..2db9251e49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -112,6 +112,8 @@ public class ModelConstants { public static final String TENANT_TITLE_PROPERTY = TITLE_PROPERTY; public static final String TENANT_REGION_PROPERTY = "region"; public static final String TENANT_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; + public static final String TENANT_ISOLATED_TB_CORE = "isolated_tb_core"; + public static final String TENANT_ISOLATED_TB_RULE_ENGINE = "isolated_tb_rule_engine"; public static final String TENANT_BY_REGION_AND_SEARCH_TEXT_COLUMN_FAMILY_NAME = "tenant_by_region_and_search_text"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java index 29a51e2539..cae31c2a31 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/TenantEntity.java @@ -24,6 +24,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.model.type.JsonCodec; @@ -55,16 +56,16 @@ public final class TenantEntity implements SearchTextEntity { @Column(name = TENANT_TITLE_PROPERTY) private String title; - + @Column(name = SEARCH_TEXT_PROPERTY) private String searchText; @Column(name = TENANT_REGION_PROPERTY) private String region; - + @Column(name = COUNTRY_PROPERTY) private String country; - + @Column(name = STATE_PROPERTY) private String state; @@ -89,6 +90,12 @@ public final class TenantEntity implements SearchTextEntity { @Column(name = TENANT_ADDITIONAL_INFO_PROPERTY, codec = JsonCodec.class) private JsonNode additionalInfo; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + public TenantEntity() { super(); } @@ -108,6 +115,8 @@ public final class TenantEntity implements SearchTextEntity { this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } public UUID getUuid() { @@ -206,6 +215,22 @@ public final class TenantEntity implements SearchTextEntity { this.additionalInfo = additionalInfo; } + public boolean isIsolatedTbCore() { + return isolatedTbCore; + } + + public void setIsolatedTbCore(boolean isolatedTbCore) { + this.isolatedTbCore = isolatedTbCore; + } + + public boolean isIsolatedTbRuleEngine() { + return isolatedTbRuleEngine; + } + + public void setIsolatedTbRuleEngine(boolean isolatedTbRuleEngine) { + this.isolatedTbRuleEngine = isolatedTbRuleEngine; + } + @Override public String getSearchTextSource() { return getTitle(); @@ -215,7 +240,7 @@ public final class TenantEntity implements SearchTextEntity { public void setSearchText(String searchText) { this.searchText = searchText; } - + public String getSearchText() { return searchText; } @@ -235,6 +260,8 @@ public final class TenantEntity implements SearchTextEntity { tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java index 10b00f38ea..1da71224f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantEntity.java @@ -72,6 +72,12 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.EMAIL_PROPERTY) private String email; + @Column(name = ModelConstants.TENANT_ISOLATED_TB_CORE) + private boolean isolatedTbCore; + + @Column(name = ModelConstants.TENANT_ISOLATED_TB_RULE_ENGINE) + private boolean isolatedTbRuleEngine; + @Type(type = "json") @Column(name = ModelConstants.TENANT_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; @@ -95,6 +101,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT this.phone = tenant.getPhone(); this.email = tenant.getEmail(); this.additionalInfo = tenant.getAdditionalInfo(); + this.isolatedTbCore = tenant.isIsolatedTbCore(); + this.isolatedTbRuleEngine = tenant.isIsolatedTbRuleEngine(); } @Override @@ -126,6 +134,8 @@ public final class TenantEntity extends BaseSqlEntity implements SearchT tenant.setPhone(phone); tenant.setEmail(email); tenant.setAdditionalInfo(additionalInfo); + tenant.setIsolatedTbCore(isolatedTbCore); + tenant.setIsolatedTbRuleEngine(isolatedTbRuleEngine); return tenant; } diff --git a/dao/src/main/resources/cassandra/schema-entities.cql b/dao/src/main/resources/cassandra/schema-entities.cql index de2b088cef..6d23ca8122 100644 --- a/dao/src/main/resources/cassandra/schema-entities.cql +++ b/dao/src/main/resources/cassandra/schema-entities.cql @@ -110,6 +110,8 @@ CREATE TABLE IF NOT EXISTS thingsboard.tenant ( phone text, email text, additional_info text, + isolated_tb_core boolean, + isolated_tb_rule_engine boolean, PRIMARY KEY (id, region) ); diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 758aaafb10..f28f7f5ebd 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -183,7 +183,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 55893fc124..36dac1b40e 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -183,7 +183,9 @@ CREATE TABLE IF NOT EXISTS tenant ( search_text varchar(255), state varchar(255), title varchar(255), - zip varchar(255) + zip varchar(255), + isolated_tb_core boolean, + isolated_tb_rule_engine boolean ); CREATE TABLE IF NOT EXISTS user_credentials ( diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index a0750164ca..7bb6e6e5e7 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1508,7 +1508,11 @@ "idCopiedMessage": "Tenant Id has been copied to clipboard", "select-tenant": "Select tenant", "no-tenants-matching": "No tenants matching '{{entity}}' were found.", - "tenant-required": "Tenant is required" + "tenant-required": "Tenant is required", + "isolated-tb-core": "Processing in isolated ThingsBoard Core container", + "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", + "isolated-tb-core-details": "Requires separate microservice(s) per isolated Tenant", + "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index aa7751d479..35b3dee289 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -15,32 +15,50 @@ limitations under the License. --> -{{ 'tenant.manage-tenant-admins' | translate }} -{{ 'tenant.delete' | translate }} +{{ + 'tenant.manage-tenant-admins' | translate }} + +{{ 'tenant.delete' + | translate }} +
- - - tenant.copyId - + + + tenant.copyId +
-
- - - -
-
tenant.title-required
-
- - - - - - -
+
+ + + +
+
tenant.title-required
+
+ + + + + + + + + {{'tenant.isolated-tb-core' | translate}}
+ {{'tenant.isolated-tb-core-details' | translate}} + + + + + {{'tenant.isolated-tb-rule-engine' | translate}}
+ {{'tenant.isolated-tb-rule-engine-details' | translate}} + + +
From 5b8ce2fb69f18486643c195cdbd241a5e5044ffd Mon Sep 17 00:00:00 2001 From: "hamza.slama" Date: Tue, 7 Apr 2020 09:26:06 +0100 Subject: [PATCH 061/196] Move AlarmId To Id Package --- .../org/thingsboard/server/controller/AlarmController.java | 2 +- .../org/thingsboard/server/controller/BaseController.java | 2 +- .../java/org/thingsboard/server/dao/alarm/AlarmService.java | 2 +- .../java/org/thingsboard/server/common/data/alarm/Alarm.java | 1 + .../thingsboard/server/common/data/{alarm => id}/AlarmId.java | 4 +--- .../thingsboard/server/common/data/id/EntityIdFactory.java | 1 - .../org/thingsboard/server/dao/alarm/BaseAlarmService.java | 2 +- .../org/thingsboard/server/dao/entity/BaseEntityService.java | 2 +- .../org/thingsboard/server/dao/model/nosql/AlarmEntity.java | 2 +- .../org/thingsboard/server/dao/model/sql/AlarmEntity.java | 2 +- .../org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java | 2 +- .../src/main/java/org/thingsboard/rest/client/RestClient.java | 3 +-- .../engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java | 2 +- .../rule/engine/util/EntitiesFieldsAsyncLoader.java | 2 +- .../rule/engine/util/EntitiesTenantIdAsyncLoader.java | 2 +- 15 files changed, 14 insertions(+), 17 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/{alarm => id}/AlarmId.java (88%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index 93754961d6..2132675f97 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index cffbce2d03..fbccaedb9e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index e2277497a5..3132aa99c7 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.alarm; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java index a43d95975e..9c23a60c28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/Alarm.java @@ -23,6 +23,7 @@ import lombok.Data; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java rename to common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java index 532842ae75..691d89cafd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/AlarmId.java @@ -13,14 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.alarm; +package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.UUIDBased; import java.util.UUID; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index c367355cb0..11db1da1a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -16,7 +16,6 @@ package org.thingsboard.server.common.data.id; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.alarm.AlarmId; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 453f1ea0be..41f8a2b630 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -29,7 +29,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 5867e505b6..7080722056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -23,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java index d2e906121f..819daa800b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/nosql/AlarmEntity.java @@ -27,7 +27,7 @@ import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java index fb4efd25d9..8d29800022 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmEntity.java @@ -26,7 +26,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index f02ae6b577..e0244c256e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -19,7 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmStatus; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 7e0dbf3858..c19de55307 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -98,7 +98,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java index 40a3e8effd..31fd207c7c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesAlarmOriginatorIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; public class EntitiesAlarmOriginatorIdAsyncLoader { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java index a0a1c8629f..182a000b59 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesFieldsAsyncLoader.java @@ -22,7 +22,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityFieldsData; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java index 3ff25e1e8b..017bacfd72 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesTenantIdAsyncLoader.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.alarm.AlarmId; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; From 6c4b50a380a2e3ac5d01ea7640eb40cc5df800aa Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Apr 2020 14:59:48 +0300 Subject: [PATCH 062/196] Implemented dedicated RE per tenant --- .../server/actors/app/AppActor.java | 28 +++++-- .../server/actors/tenant/TenantActor.java | 25 +++++-- .../server/controller/TenantController.java | 2 - .../queue/DefaultTbClusterService.java | 8 ++ .../DefaultTenantRoutingInfoService.java | 47 ++++++++++++ .../DefaultSubscriptionManagerService.java | 28 +++---- .../transport/DefaultTransportApiService.java | 19 ++++- application/src/main/resources/logback.xml | 2 + .../src/main/resources/thingsboard.yml | 27 +++---- .../ConsistentHashParitionServiceTest.java | 5 +- .../ConsistentHashPartitionService.java | 43 ++++++++--- .../queue/discovery/TenantRoutingInfo.java | 26 +++++++ .../discovery/TenantRoutingInfoService.java | 23 ++++++ .../TbQueueRemoteJsInvokeSettings.java | 12 +-- .../server/queue/util/TbCoreComponent.java | 4 + .../queue/util/TbRuleEngineComponent.java | 4 + common/queue/src/main/proto/queue.proto | 12 +++ .../common/transport/TransportService.java | 14 +++- .../service/DefaultTransportService.java | 32 ++++++++ .../TransportTenantRoutingInfoService.java | 52 +++++++++++++ .../server/dao/tenant/TenantServiceImpl.java | 35 ++++++--- .../src/main/resources/tb-mqtt-transport.yml | 74 ++++++++++++------- 22 files changed, 420 insertions(+), 102 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 145ae10d06..7e4773aa8b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -39,10 +39,13 @@ import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import scala.concurrent.duration.Duration; +import java.util.Optional; + public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); @@ -68,7 +71,7 @@ public class AppActor extends ContextAwareActor { @Override protected boolean process(TbActorMsg msg) { if (!ruleChainsInitialized) { - initRuleChainsAndTenantActors(); + initTenantActors(); ruleChainsInitialized = true; if (msg.getMsgType() != MsgType.APP_INIT_MSG) { log.warn("Rule Chains initialized by unexpected message: {}", msg); @@ -100,15 +103,30 @@ public class AppActor extends ContextAwareActor { return true; } - private void initRuleChainsAndTenantActors() { + private void initTenantActors() { log.info("Starting main system actor."); try { - if (systemContext.isTenantComponentsInitEnabled()) { - PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - for (Tenant tenant : tenantIterator) { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (isolatedTenantId.isPresent()) { + Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get()); + if (tenant != null) { log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("Tenant actor created."); + } else { + log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get()); + } + } else if (systemContext.isTenantComponentsInitEnabled()) { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + for (Tenant tenant : tenantIterator) { + if (isCore || (isRuleEngine && !tenant.isIsolatedTbRuleEngine())) { + log.debug("[{}] Creating tenant actor", tenant.getId()); + getOrCreateTenantActor(tenant.getId()); + log.debug("[{}] Tenant actor created.", tenant.getId()); + } } } log.info("Main system actor started."); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index e54b955419..a1bf343efe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -22,7 +22,6 @@ import akka.actor.OneForOneStrategy; import akka.actor.Props; import akka.actor.SupervisorStrategy; import akka.actor.Terminated; -import akka.japi.Function; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import org.thingsboard.server.actors.ActorSystemContext; @@ -31,6 +30,7 @@ import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -47,12 +47,13 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import scala.concurrent.duration.Duration; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; public class TenantActor extends RuleChainManagerActor { private final BiMap deviceActors; - private boolean isRuleEngine; + private boolean isRuleEngineForCurrentTenant; private boolean isCore; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { @@ -69,10 +70,19 @@ public class TenantActor extends RuleChainManagerActor { public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); try { - isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + Tenant tenant = systemContext.getTenantService().findTenantById(tenantId); + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - if (isRuleEngine) { - initRuleChains(); + + if (isRuleEngineForCurrentTenant) { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } } log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { @@ -132,8 +142,9 @@ public class TenantActor extends RuleChainManagerActor { } private void onQueueToRuleEngineMsg(QueueToRuleEngineMsg msg) { - if (!isRuleEngine) { + if (!isRuleEngineForCurrentTenant) { log.warn("RECEIVED INVALID MESSAGE: {}", msg); + return; } TbMsg tbMsg = msg.getTbMsg(); if (tbMsg.getRuleChainId() == null) { @@ -167,7 +178,7 @@ public class TenantActor extends RuleChainManagerActor { } private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { - if (isRuleEngine) { + if (isRuleEngineForCurrentTenant) { ActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index abd6bf6804..9def943e88 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -76,7 +76,6 @@ public class TenantController extends BaseController { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, tenant.getId(), tenant); - tenant = checkNotNull(tenantService.saveTenant(tenant)); if (newTenant) { installScripts.createDefaultRuleChains(tenant.getId()); @@ -96,7 +95,6 @@ public class TenantController extends BaseController { TenantId tenantId = new TenantId(toUUID(strTenantId)); checkTenantId(tenantId, Operation.DELETE); tenantService.deleteTenant(tenantId); - tbClusterService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5851fd919c..fa6aa9e886 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -54,6 +54,14 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java new file mode 100644 index 0000000000..aae3cef0ac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") +public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { + + private final TenantService tenantService; + + public DefaultTenantRoutingInfoService(TenantService tenantService) { + this.tenantService = tenantService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + Tenant tenant = tenantService.findTenantById(tenantId); + if (tenant != null) { + return new TenantRoutingInfo(tenantId, tenant.isIsolatedTbCore(), tenant.isIsolatedTbRuleEngine()); + } else { + throw new RuntimeException("Tenant not found!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index 0d151c3f46..e841acae12 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -173,19 +173,21 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer @Override public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - Set removedPartitions = new HashSet<>(currentPartitions); - removedPartitions.removeAll(partitionChangeEvent.getPartitions()); - - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - - // We no longer manage current partition of devices; - removedPartitions.forEach(partition -> { - Set subs = partitionedSubscriptions.remove(partition); - if (subs != null) { - subs.forEach(this::removeSubscriptionFromEntityMap); - } - }); + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + Set removedPartitions = new HashSet<>(currentPartitions); + removedPartitions.removeAll(partitionChangeEvent.getPartitions()); + + currentPartitions.clear(); + currentPartitions.addAll(partitionChangeEvent.getPartitions()); + + // We no longer manage current partition of devices; + removedPartitions.forEach(partition -> { + Set subs = partitionedSubscriptions.remove(partition); + if (subs != null) { + subs.forEach(this::removeSubscriptionFromEntityMap); + } + }); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 4ff647b4d9..7456ca7662 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -24,8 +24,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -34,14 +34,18 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.state.DeviceStateService; @@ -59,6 +63,10 @@ public class DefaultTransportApiService implements TransportApiService { private static final ObjectMapper mapper = new ObjectMapper(); + //TODO: Constructor dependencies; + @Autowired + private TenantService tenantService; + @Autowired private DeviceService deviceService; @@ -87,6 +95,8 @@ public class DefaultTransportApiService implements TransportApiService { return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + } else if (transportApiRequestMsg.hasGetTenantRoutingInfoRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getGetTenantRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } @@ -129,6 +139,13 @@ public class DefaultTransportApiService implements TransportApiService { }, dbCallbackExecutorService); } + private ListenableFuture handle(GetTenantRoutingInfoRequestMsg requestMsg) { + TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); + ListenableFuture tenantFuture = tenantService.findTenantByIdAsync(TenantId.SYS_TENANT_ID, tenantId); + return Futures.transform(tenantFuture, tenant -> TransportApiResponseMsg.newBuilder() + .setGetTenantRoutingInfoResponseMsg(GetTenantRoutingInfoResponseMsg.newBuilder().setIsolatedTbCore(tenant.isIsolatedTbCore()) + .setIsolatedTbRuleEngine(tenant.isIsolatedTbRuleEngine()).build()).build(), dbCallbackExecutorService); + } private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 8389fa5d71..08a976592d 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index affd7e7908..a2a06b57d0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -437,18 +437,6 @@ js: print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - # JS response auto commit interval - response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted max_errors: "${REMOTE_JS_SANDBOX_MAX_ERRORS:3}" # Maximum time in seconds for black listed function to stay in the list. @@ -579,6 +567,19 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" @@ -586,7 +587,7 @@ queue: stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: # TODO 2.5: specify correct ENV variable names. + queues: - name: "Main" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 2634d5af99..3f7619ed1e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import java.util.ArrayList; import java.util.Collections; @@ -49,6 +50,7 @@ public class ConsistentHashParitionServiceTest { private ConsistentHashPartitionService clusterRoutingService; private TbServiceInfoProvider discoveryService; + private TenantRoutingInfoService routingInfoService; private ApplicationEventPublisher applicationEventPublisher; private String hashFunctionName = "murmur3_128"; @@ -59,7 +61,8 @@ public class ConsistentHashParitionServiceTest { public void setup() throws Exception { discoveryService = mock(TbServiceInfoProvider.class); applicationEventPublisher = mock(ApplicationEventPublisher.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService, applicationEventPublisher); + routingInfoService = mock(TenantRoutingInfoService.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index 0552aa05c6..dfd1437b5f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -60,11 +60,12 @@ public class ConsistentHashPartitionService implements PartitionService { private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; + private final TenantRoutingInfoService tenantRoutingInfoService; private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); + private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); - //TODO: Fetch this from the database, together with size of partitions for each service for each tenant. - private ConcurrentMap> isolatedTenants = new ConcurrentHashMap<>(); private ConcurrentMap tpiCache = new ConcurrentHashMap<>(); private Map tbCoreNotificationTopics = new HashMap<>(); @@ -73,8 +74,9 @@ public class ConsistentHashPartitionService implements PartitionService { private HashFunction hashFunction; - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, ApplicationEventPublisher applicationEventPublisher) { + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { this.serviceInfoProvider = serviceInfoProvider; + this.tenantRoutingInfoService = tenantRoutingInfoService; this.applicationEventPublisher = applicationEventPublisher; } @@ -122,10 +124,10 @@ public class ConsistentHashPartitionService implements PartitionService { addNode(circles, other); } ConcurrentMap> oldPartitions = myPartitions; - TenantId myTenantId = getSystemOrIsolatedTenantId(currentService); + TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); myPartitions = new ConcurrentHashMap<>(); partitionSizes.forEach((type, size) -> { - ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myTenantId); + ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(type, myIsolatedOrSystemTenantId); for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(circles.get(myServiceQueueKey), i); if (currentService.equals(serviceInfo)) { @@ -247,7 +249,30 @@ public class ConsistentHashPartitionService implements PartitionService { } private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) { - return isolatedTenants.get(tenantId) != null && isolatedTenants.get(tenantId).contains(serviceQueue.getType()); + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return false; + } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + synchronized (tenantRoutingInfoMap) { + routingInfo = tenantRoutingInfoMap.get(tenantId); + if (routingInfo == null) { + routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + tenantRoutingInfoMap.put(tenantId, routingInfo); + } + } + } + if (routingInfo == null) { + throw new RuntimeException("Tenant not found!"); + } + switch (serviceQueue.getType()) { + case TB_CORE: + return routingInfo.isIsolatedTbCore(); + case TB_RULE_ENGINE: + return routingInfo.isIsolatedTbRuleEngine(); + default: + return false; + } } private void logServiceInfo(TransportProtos.ServiceInfo server) { @@ -265,12 +290,6 @@ public class ConsistentHashPartitionService implements PartitionService { private void addNode(Map> circles, ServiceInfo instance) { TenantId tenantId = getSystemOrIsolatedTenantId(instance); - if (!tenantId.isNullUid()) { - isolatedTenants.putIfAbsent(tenantId, new HashSet<>()); - for (String serviceType : instance.getServiceTypesList()) { - isolatedTenants.get(tenantId).add(ServiceType.valueOf(serviceType.toUpperCase())); - } - } for (String serviceTypeStr : instance.getServiceTypesList()) { ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase()); if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java new file mode 100644 index 0000000000..07c4c2d9c5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class TenantRoutingInfo { + private final TenantId tenantId; + private final boolean isolatedTbCore; + private final boolean isolatedTbRuleEngine; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java new file mode 100644 index 0000000000..685a36df30 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface TenantRoutingInfoService { + + TenantRoutingInfo getRoutingInfo(TenantId tenantId); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index cdad407b19..cd492e87f1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -22,21 +22,21 @@ import org.springframework.stereotype.Component; @Data @Component public class TbQueueRemoteJsInvokeSettings { - @Value("${js.remote.request_topic}") + @Value("${queue.js.request_topic}") private String requestTopic; - @Value("${js.remote.response_topic_prefix}") + @Value("${queue.js.response_topic_prefix}") private String responseTopic; - @Value("${js.remote.max_pending_requests}") + @Value("${queue.js.max_pending_requests}") private long maxPendingRequests; - @Value("${js.remote.response_poll_interval}") + @Value("${queue.js.response_poll_interval}") private int responsePollInterval; - @Value("${js.remote.response_auto_commit_interval}") + @Value("${queue.js.response_auto_commit_interval}") private int autoCommitInterval; - @Value("${js.remote.max_requests_timeout}") + @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java index c42b9b9f7d..ca2316e174 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbCoreComponent.java @@ -17,6 +17,10 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core'") public @interface TbCoreComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java index 855bebb19e..3b99c94dea 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbRuleEngineComponent.java @@ -17,6 +17,10 @@ package org.thingsboard.server.queue.util; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine'") public @interface TbRuleEngineComponent { } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index e36c4e36a9..3db8495d45 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -158,6 +158,16 @@ message GetOrCreateDeviceFromGatewayResponseMsg { DeviceInfoProto deviceInfo = 1; } +message GetTenantRoutingInfoRequestMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; +} + +message GetTenantRoutingInfoResponseMsg { + bool isolatedTbCore = 1; + bool isolatedTbRuleEngine = 2; +} + message SessionCloseNotificationProto { string message = 1; } @@ -359,12 +369,14 @@ message TransportApiRequestMsg { ValidateDeviceTokenRequestMsg validateTokenRequestMsg = 1; ValidateDeviceX509CertRequestMsg validateX509CertRequestMsg = 2; GetOrCreateDeviceFromGatewayRequestMsg getOrCreateDeviceRequestMsg = 3; + GetTenantRoutingInfoRequestMsg getTenantRoutingInfoRequestMsg = 4; } /* Response from ThingsBoard Core Service to Transport Service */ message TransportApiResponseMsg { ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1; GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; + GetTenantRoutingInfoResponseMsg getTenantRoutingInfoResponseMsg = 4; } /* Messages that are handled by ThingsBoard Core Service */ diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index b0772de945..2af55c9206 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.common.transport; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -35,14 +39,16 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce */ public interface TransportService { + GetTenantRoutingInfoResponseMsg getRoutingInfo(GetTenantRoutingInfoRequestMsg msg); + void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg, - TransportServiceCallback callback); + void process(GetOrCreateDeviceFromGatewayRequestMsg msg, + TransportServiceCallback callback); boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback); @@ -62,7 +68,7 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback callback); - void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback); + void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback callback); void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index d107bbbeeb..7b2f157cb4 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.transport.service; import com.google.gson.Gson; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -43,6 +44,8 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.gen.transport.TransportProtos; @@ -61,6 +64,7 @@ import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -194,6 +198,34 @@ public class DefaultTransportService implements TransportService { sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } + @Override + public TransportProtos.GetTenantRoutingInfoResponseMsg getRoutingInfo(TransportProtos.GetTenantRoutingInfoRequestMsg msg) { + TbProtoQueueMsg protoMsg = + new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getGetTenantRoutingInfoResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + + TransportProtos.GetTenantRoutingInfoRequestMsg msg = TransportProtos.GetTenantRoutingInfoRequestMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + TransportProtos.GetTenantRoutingInfoResponseMsg routingInfo = response.getValue().getGetTenantRoutingInfoResponseMsg(); + return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + @Override public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java new file mode 100644 index 0000000000..f2534a64c9 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetTenantRoutingInfoResponseMsg; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Slf4j +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-transport'") +public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { + + private TransportService transportService; + + @Lazy + @Autowired + public void setTransportService(TransportService transportService) { + this.transportService = transportService; + } + + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + GetTenantRoutingInfoRequestMsg msg = GetTenantRoutingInfoRequestMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + GetTenantRoutingInfoResponseMsg routingInfo = transportService.getRoutingInfo(msg); + return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 9453fdb4c5..eb916cdb83 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; @@ -126,7 +127,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public void deleteTenants() { log.trace("Executing deleteTenants"); - tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID),DEFAULT_TENANT_REGION); + tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); } private DataValidator tenantValidator = @@ -140,19 +141,31 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe validateEmail(tenant.getEmail()); } } - }; + + @Override + protected void validateUpdate(TenantId tenantId, Tenant tenant) { + Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing tenant!"); + } else if (old.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { + throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); + } else if (old.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { + throw new DataValidationException("Can't update isolatedTbCore property!"); + } + } + }; private PaginatedRemover tenantsRemover = new PaginatedRemover() { - @Override - protected List findEntities(TenantId tenantId, String region, TextPageLink pageLink) { - return tenantDao.findTenantsByRegion(tenantId, region, pageLink); - } + @Override + protected List findEntities(TenantId tenantId, String region, TextPageLink pageLink) { + return tenantDao.findTenantsByRegion(tenantId, region, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(new TenantId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(new TenantId(entity.getUuidId())); + } + }; } diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 395f1d09e0..c421c43e33 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -63,7 +63,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -111,12 +111,25 @@ queue: response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" - poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack_processing_timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" - print_interval_ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" @@ -124,32 +137,39 @@ queue: stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" - queues: # TODO 2.5: specify correct ENV variable names. - - - name: "Main" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.main}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - - - name: "HighPriority" - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine.hp}" - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RULE_ENGINE_PARTITIONS:3}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" - ack-strategy: - type: "${TB_QUEUE_RULE_ENGINE_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT - retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited - failure-percentage: "${TB_QUEUE_RULE_ENGINE_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; - pause-between-retries: "${TB_QUEUE_RULE_ENGINE_STRATEGY_RETRY_PAUSE:1}"# Time in seconds to wait in consumer thread before retries; + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: + # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" From 006fcd03ed09eff402074c33bc2426f7b1d0f706 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Apr 2020 15:01:20 +0300 Subject: [PATCH 063/196] Improved configuration for HTTP and CoAP transport --- .../src/main/resources/tb-coap-transport.yml | 135 +++++++++++++++--- .../src/main/resources/tb-http-transport.yml | 135 +++++++++++++++--- 2 files changed, 230 insertions(+), 40 deletions(-) diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 9b3572c7d1..70d1ff8fbe 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -41,24 +41,119 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 4097bce831..5ddb7c0553 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -42,24 +42,119 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" -kafka: - enabled: true - bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" - acks: "${TB_KAFKA_ACKS:all}" - retries: "${TB_KAFKA_RETRIES:1}" - batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" - linger.ms: "${TB_KAFKA_LINGER_MS:1}" - buffer.memory: "${TB_BUFFER_MEMORY:33554432}" +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + aws_sqs: + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" - max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - response_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - response_auto_commit_interval: "${TB_TRANSPORT_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" - rule_engine: - topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}" - notifications: - topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" - poll_interval: "${TB_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - auto_commit_interval: "${TB_TRANSPORT_NOTIFICATIONS_AUTO_COMMIT_INTERVAL_MS:100}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + # JS response auto commit interval + response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" + queues: + - name: "Main" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. \ No newline at end of file From d380ac710646a2aea8693ea14aba1eaa4b9f4994 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Apr 2020 18:33:11 +0300 Subject: [PATCH 064/196] Device -> Server RPC implementation and improvements --- .../server/actors/ActorSystemContext.java | 4 - .../server/actors/device/DeviceActor.java | 9 -- .../device/DeviceActorMessageProcessor.java | 108 +++++-------- .../queue/DefaultTbClusterService.java | 6 + .../service/queue/TbClusterService.java | 3 + .../rpc/DefaultTbRuleEngineRpcService.java | 14 +- .../rpc/ToServerRpcResponseActorMsg.java | 47 ------ .../DefaultTbCoreToTransportService.java | 27 ++-- .../transport/TbCoreToTransportService.java | 6 +- .../src/main/resources/thingsboard.yml | 22 +-- .../server/common/msg/MsgType.java | 2 - .../DeviceActorClientSideRpcTimeoutMsg.java | 33 ---- .../ConsistentHashPartitionService.java | 5 - .../queue/discovery/PartitionService.java | 2 - .../queue/kafka/TBKafkaProducerTemplate.java | 2 +- common/queue/src/main/proto/queue.proto | 18 +-- .../transport/coap/CoapTransportResource.java | 2 + .../transport/http/DeviceApiController.java | 2 + .../transport/mqtt/MqttTransportHandler.java | 2 + .../mqtt/session/GatewayDeviceSessionCtx.java | 2 + .../service/DefaultTransportService.java | 144 +++++++++++------- .../service/RpcRequestMetadata.java} | 16 +- .../rule/engine/api/RuleEngineRpcService.java | 4 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 10 +- .../rpc/TbSendRpcReplyNodeConfiguration.java | 24 ++- .../src/main/resources/tb-coap-transport.yml | 18 +-- .../src/main/resources/tb-http-transport.yml | 18 +-- .../src/main/resources/tb-mqtt-transport.yml | 36 +++-- 28 files changed, 266 insertions(+), 320 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java rename common/{message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java => transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java} (76%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index d31cb62b9b..7822438724 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -270,10 +270,6 @@ public class ActorSystemContext { @Getter private long queuePersistenceTimeout; - @Value("${actors.client_side_rpc.timeout}") - @Getter - private long clientSideRpcTimeout; - @Value("${actors.rule.chain.error_persist_frequency}") @Getter private long ruleChainErrorPersistFrequency; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 3a15240f33..256ffd0b30 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -22,11 +22,8 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; -import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; public class DeviceActor extends ContextAwareActor { @@ -72,15 +69,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG: processor.processRpcRequest(context(), (ToDeviceRpcRequestActorMsg) msg); break; - case SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG: - processor.processToServerRPCResponse(context(), (ToServerRpcResponseActorMsg) msg); - break; case DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG: processor.processServerSideRpcTimeout(context(), (DeviceActorServerSideRpcTimeoutMsg) msg); break; - case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG: - processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg); - break; case SESSION_TIMEOUT_MSG: processor.checkSessionsTimeout(); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 83d269ef6d..437ed456e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -36,12 +36,11 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; -import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; @@ -50,16 +49,19 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotifica import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent; import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionSubscriptionInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.SessionType; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToAttributeUpdatesMsg; import org.thingsboard.server.gen.transport.TransportProtos.SubscribeToRPCMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.service.rpc.ToServerRpcResponseActorMsg; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.Nullable; @@ -91,7 +93,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private final Map attributeSubscriptions; private final Map rpcSubscriptions; private final Map toDeviceRpcPendingMap; - private final Map toServerRpcPendingMap; private int rpcSeq = 0; private String deviceName; @@ -106,7 +107,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { this.attributeSubscriptions = new HashMap<>(); this.rpcSubscriptions = new HashMap<>(); this.toDeviceRpcPendingMap = new HashMap<>(); - this.toServerRpcPendingMap = new HashMap<>(); if (initAttributes()) { restoreSessions(); } @@ -142,7 +142,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { Set syncSessionSet = new HashSet<>(); rpcSubscriptions.forEach((key, value) -> { sendToTransport(rpcRequest, key, value.getNodeId()); - if (TransportProtos.SessionType.SYNC == value.getType()) { + if (SessionType.SYNC == value.getType()) { syncSessionSet.add(key); } }); @@ -177,10 +177,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) { - TransportProtos.SessionType sessionType = getSessionType(sessionId); + SessionType sessionType = getSessionType(sessionId); if (!toDeviceRpcPendingMap.isEmpty()) { log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId); - if (sessionType == TransportProtos.SessionType.SYNC) { + if (sessionType == SessionType.SYNC) { log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId); rpcSubscriptions.remove(sessionId); } @@ -188,7 +188,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId); } Set sentOneWayIds = new HashSet<>(); - if (sessionType == TransportProtos.SessionType.ASYNC) { + if (sessionType == SessionType.ASYNC) { toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); } else { toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds)); @@ -297,46 +297,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return new HashSet<>(strings); } - private void handleClientSideRPCRequest(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg request) { -// UUID sessionId = getSessionId(sessionInfo); -// JsonObject json = new JsonObject(); -// json.addProperty("method", request.getMethodName()); -// json.add("params", JsonUtils.parse(request.getParams())); -// -// TbMsgMetaData requestMetaData = defaultMetaData.copy(); -// requestMetaData.putValue("requestId", Integer.toString(request.getRequestId())); -// TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, requestMetaData, TbMsgDataType.JSON, gson.toJson(json), null, null, null); -// context.parent().tell(new DeviceActorToRuleEngineMsg(context.self(), tbMsg), context.self()); -// -// scheduleMsgWithDelay(context, new DeviceActorClientSideRpcTimeoutMsg(request.getRequestId(), systemContext.getClientSideRpcTimeout()), systemContext.getClientSideRpcTimeout()); -// toServerRpcPendingMap.put(request.getRequestId(), new ToServerRpcRequestMetadata(sessionId, getSessionType(sessionId), sessionInfo.getNodeId())); - } - - private TransportProtos.SessionType getSessionType(UUID sessionId) { - return sessions.containsKey(sessionId) ? TransportProtos.SessionType.ASYNC : TransportProtos.SessionType.SYNC; - } - - void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) { - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId()); - if (data != null) { - log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(msg.getId()).setError("timeout").build() - , data.getSessionId(), data.getNodeId()); - } - } - - void processToServerRPCResponse(ActorContext context, ToServerRpcResponseActorMsg msg) { - int requestId = msg.getMsg().getRequestId(); - ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId()); - sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder() - .setRequestId(requestId).setPayload(msg.getMsg().getData()).build() - , data.getSessionId(), data.getNodeId()); - } else { - log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId); - } + private SessionType getSessionType(UUID sessionId) { + return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC; } void processAttributesUpdate(ActorContext context, DeviceAttributesEventNotificationMsg msg) { @@ -399,7 +361,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToAttributes(true); log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId); @@ -420,7 +382,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } else { SessionInfoMetaData sessionMD = sessions.get(sessionId); if (sessionMD == null) { - sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId())); + sessionMD = new SessionInfoMetaData(new SessionInfo(SessionType.SYNC, sessionInfo.getNodeId())); } sessionMD.setSubscribedToRPC(true); log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId); @@ -444,7 +406,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove)); } } - sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()))); + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfo.getNodeId()))); if (sessions.size() == 1) { reportSessionOpen(); } @@ -462,10 +424,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) { + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, SubscriptionInfoProto subscriptionInfo) { UUID sessionId = getSessionId(sessionInfoProto); SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId, - id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); + id -> new SessionInfoMetaData(new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L)); sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime()); sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription()); @@ -488,7 +450,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build(); @@ -504,7 +466,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(GetAttributeResponseMsg responseMsg, SessionInfoProto sessionInfo) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionInfo.getSessionIdMSB()) .setSessionIdLSB(sessionInfo.getSessionIdLSB()) .setGetAttributesResponse(responseMsg).build(); @@ -512,7 +474,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(AttributeUpdateNotificationMsg notificationMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setAttributeUpdateNotification(notificationMsg).build(); @@ -520,15 +482,15 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } private void sendToTransport(ToDeviceRpcRequestMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToDeviceRequest(rpcMsg).build(); systemContext.getTbCoreToTransportService().process(nodeId, msg); } - private void sendToTransport(TransportProtos.ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { - DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder() + private void sendToTransport(ToServerRpcResponseMsg rpcMsg, UUID sessionId, String nodeId) { + ToTransportMsg msg = ToTransportMsg.newBuilder() .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(rpcMsg).build(); @@ -584,9 +546,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void restoreSessions() { log.debug("[{}] Restoring sessions from cache", deviceId); - TransportProtos.DeviceSessionsCacheEntry sessionsDump = null; + DeviceSessionsCacheEntry sessionsDump = null; try { - sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); + sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); } catch (InvalidProtocolBufferException e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; @@ -595,11 +557,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { log.debug("[{}] No session information found", deviceId); return; } - for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { + for (SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) { SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo(); UUID sessionId = getSessionId(sessionInfoProto); - SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()); - TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); + SessionInfo sessionInfo = new SessionInfo(SessionType.ASYNC, sessionInfoProto.getNodeId()); + SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo(); SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime()); sessions.put(sessionId, sessionMD); if (subInfo.getAttributeSubscription()) { @@ -617,27 +579,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { private void dumpSessions() { log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size()); - List sessionsList = new ArrayList<>(sessions.size()); + List sessionsList = new ArrayList<>(sessions.size()); sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) { + if (sessionMD.getSessionInfo().getType() == SessionType.SYNC) { return; } SessionInfo sessionInfo = sessionMD.getSessionInfo(); - TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() + SubscriptionInfoProto subscriptionInfoProto = SubscriptionInfoProto.newBuilder() .setLastActivityTime(sessionMD.getLastActivityTime()) .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()).build(); - TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder() + SessionInfoProto sessionInfoProto = SessionInfoProto.newBuilder() .setSessionIdMSB(uuid.getMostSignificantBits()) .setSessionIdLSB(uuid.getLeastSignificantBits()) .setNodeId(sessionInfo.getNodeId()).build(); - sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder() + sessionsList.add(SessionSubscriptionInfoProto.newBuilder() .setSessionInfo(sessionInfoProto) .setSubscriptionInfo(subscriptionInfoProto).build()); log.debug("[{}] Dumping session: {}", deviceId, sessionMD); }); systemContext.getDeviceSessionCacheService() - .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder() + .put(deviceId, DeviceSessionsCacheEntry.newBuilder() .addAllSessions(sessionsList).build().toByteArray()); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index fa6aa9e886..f93657427d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import java.util.HashSet; import java.util.Set; +import java.util.UUID; @Service @Slf4j @@ -100,7 +101,12 @@ public class DefaultTbClusterService implements TbClusterService { response.getResponse().ifPresent(builder::setResponse); ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + } + @Override + public void onToTransportMsg(String serviceId, ToTransportMsg response) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), null); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java index 952b88e4d3..b70a31532e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbClusterService.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; public interface TbClusterService { @@ -32,6 +33,8 @@ public interface TbClusterService { void onToRuleEngineMsg(String targetServiceId, FromDeviceRpcResponse response); + void onToTransportMsg(String targetServiceId, ToTransportMsg response); + void onEntityStateChange(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index c383da3a79..909690ff6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -52,6 +53,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi private final TbClusterService clusterService; private final TbServiceInfoProvider serviceInfoProvider; + private final ConcurrentMap> toDeviceRpcRequests = new ConcurrentHashMap<>(); private Optional tbCoreRpcService; @@ -85,8 +87,16 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi } @Override - public void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body) { -// TODO 2.5 + public void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body) { + TransportProtos.ToServerRpcResponseMsg responseMsg = TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(requestId) + .setPayload(body).build(); + TransportProtos.ToTransportMsg msg = TransportProtos.ToTransportMsg.newBuilder() + .setSessionIdMSB(sessionId.getMostSignificantBits()) + .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setToServerResponse(responseMsg) + .build(); + clusterService.onToTransportMsg(serviceId, msg); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java b/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java deleted file mode 100644 index 570112dab3..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/rpc/ToServerRpcResponseActorMsg.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.rpc; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.ToString; -import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.core.ToServerRpcResponseMsg; - -/** - * Created by ashvayka on 16.04.18. - */ -@ToString -@RequiredArgsConstructor -public class ToServerRpcResponseActorMsg implements ToDeviceActorNotificationMsg { - - @Getter - private final TenantId tenantId; - - @Getter - private final DeviceId deviceId; - - @Getter - private final ToServerRpcResponseMsg msg; - - @Override - public MsgType getMsgType() { - return MsgType.SERVER_RPC_RESPONSE_TO_DEVICE_ACTOR_MSG; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java index 8f29c46466..c9e37e6791 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java @@ -18,13 +18,14 @@ package org.thingsboard.server.service.transport; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceActorToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -38,28 +39,26 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @TbCoreComponent public class DefaultTbCoreToTransportService implements TbCoreToTransportService { + private final PartitionService partitionService; private final TbQueueProducer> tbTransportProducer; - @Value("${queue.transport.notifications_topic}") - private String notificationsTopic; - - public DefaultTbCoreToTransportService(TbQueueProducerProvider tbQueueProducerProvider) { + public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) { + this.partitionService = partitionService; this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer(); } @Override - public void process(String nodeId, DeviceActorToTransportMsg msg) { + public void process(String nodeId, ToTransportMsg msg) { process(nodeId, msg, null, null); } @Override - public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { - String topic = notificationsTopic + "." + nodeId; + public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure) { + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId); UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()); - ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build(); - log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg); - TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, transportMsg); - tbTransportProducer.send(TopicPartitionInfo.builder().topic(topic).build(), queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); + log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg); + TbProtoQueueMsg queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg); + tbTransportProducer.send(tpi, queueMsg, new QueueCallbackAdaptor(onSuccess, onFailure)); } private static class QueueCallbackAdaptor implements TbQueueCallback { diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java index 5b11edb5f0..e0d5e2121a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreToTransportService.java @@ -15,14 +15,14 @@ */ package org.thingsboard.server.service.transport; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import java.util.function.Consumer; public interface TbCoreToTransportService { - void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg); + void process(String nodeId, ToTransportMsg msg); - void process(String nodeId, TransportProtos.DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer onFailure); + void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer onFailure); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a2a06b57d0..b5e875e07b 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -251,8 +251,6 @@ actors: enabled: "${ACTORS_QUEUE_ENABLED:true}" # Maximum allowed timeout for persistence into the queue timeout: "${ACTORS_QUEUE_PERSISTENCE_TIMEOUT:30000}" - client_side_rpc: - timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" cache: # caffeine or redis @@ -458,6 +456,8 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" + client_side_rpc: + timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Local HTTP transport parameters http: enabled: "${HTTP_ENABLED:true}" @@ -552,15 +552,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -569,9 +569,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -581,7 +581,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -589,7 +589,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -604,7 +604,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -620,7 +620,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 47e09b812f..0345355aa3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -89,8 +89,6 @@ public enum MsgType { DEVICE_ACTOR_SERVER_SIDE_RPC_TIMEOUT_MSG, - DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG, - /** * Message that is sent from the Device Actor to Rule Engine. Requires acknowledgement */ diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java deleted file mode 100644 index 5d47425e43..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/timeout/DeviceActorClientSideRpcTimeoutMsg.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.timeout; - -import org.thingsboard.server.common.msg.MsgType; - -/** - * @author Andrew Shvayka - */ -public final class DeviceActorClientSideRpcTimeoutMsg extends TimeoutMsg { - - public DeviceActorClientSideRpcTimeoutMsg(Integer id, long timeout) { - super(id, timeout); - } - - @Override - public MsgType getMsgType() { - return MsgType.DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG; - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index dfd1437b5f..df25ea3ba9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -196,11 +196,6 @@ public class ConsistentHashPartitionService implements PartitionService { } } - @Override - public Set getIsolatedTenants(ServiceType serviceType) { - throw new RuntimeException("Not Implemented!"); - } - private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 7c6900d8a8..e3b3e76b80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -47,8 +47,6 @@ public interface PartitionService { */ Set getAllServiceIds(ServiceType serviceType); - Set getIsolatedTenants(ServiceType serviceType); - /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. * This topic is likely to have single partition, and is always assigned to the service. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 2fbfa7d3df..2d82ed275d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -79,7 +79,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure", exception); + log.warn("Producer template failure: {}", exception.getMessage(), exception); } } }); diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 3db8495d45..dac64a3478 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -245,16 +245,6 @@ message TransportToRuleEngineMsg { ToServerRpcRequestMsg toServerRPCCallRequest = 5; } -message DeviceActorToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; -} - /** * TB Core Data Structures */ @@ -410,5 +400,11 @@ message ToRuleEngineNotificationMsg { /* Messages that are handled by ThingsBoard Transport Service */ message ToTransportMsg { - DeviceActorToTransportMsg toDeviceSessionMsg = 1; + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; } diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index 9d2cff5c09..82678fde5d 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -303,6 +303,8 @@ public class CoapTransportResource extends CoapResource { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index c8e7f76450..c2f171e39b 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -221,6 +221,8 @@ public class DeviceApiController { .setDeviceIdLSB(deviceInfoProto.getDeviceIdLSB()) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); onSuccess.accept(sessionInfo); } else { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index fb509e91af..7295cb786b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -507,6 +507,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement .setDeviceIdLSB(msg.getDeviceInfo().getDeviceIdLSB()) .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB()) .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB()) + .setDeviceName(msg.getDeviceInfo().getDeviceName()) + .setDeviceType(msg.getDeviceInfo().getDeviceType()) .build(); transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.registerAsyncSession(sessionInfo, this); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index 299f5449d9..b4b5f98238 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -44,6 +44,8 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setDeviceIdLSB(deviceInfo.getDeviceIdLSB()) .setTenantIdMSB(deviceInfo.getTenantIdMSB()) .setTenantIdLSB(deviceInfo.getTenantIdLSB()) + .setDeviceName(deviceInfo.getDeviceName()) + .setDeviceType(deviceInfo.getDeviceType()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 7b2f157cb4..389f50009f 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -18,36 +18,25 @@ package org.thingsboard.server.common.transport.service; import com.google.gson.Gson; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.session.SessionMsgType; -import org.thingsboard.server.common.transport.util.JsonUtils; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfo; -import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.provider.TbTransportQueueFactory; +import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -55,11 +44,23 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.AsyncCallbackTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -90,6 +91,8 @@ public class DefaultTransportService implements TransportService { private long sessionInactivityTimeout; @Value("${transport.sessions.report_timeout}") private long sessionReportTimeout; + @Value("${transport.client_side_rpc.timeout:60000}") + private long clientSideRpcTimeout; @Value("${queue.transport.poll_interval}") private int notificationsPollDuration; @@ -97,6 +100,7 @@ public class DefaultTransportService implements TransportService { private final TbTransportQueueFactory queueProvider; private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; + private final TbServiceInfoProvider serviceInfoProvider; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; @@ -106,15 +110,17 @@ public class DefaultTransportService implements TransportService { protected ScheduledExecutorService schedulerExecutor; protected ExecutorService transportCallbackExecutor; - private ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); //TODO: Implement cleanup of this maps. - private ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); - private ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap perDeviceLimits = new ConcurrentHashMap<>(); private ExecutorService mainConsumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("transport-consumer")); private volatile boolean stopped = false; - public DefaultTransportService(TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + public DefaultTransportService(TbServiceInfoProvider serviceInfoProvider, TbTransportQueueFactory queueProvider, TbQueueProducerProvider producerProvider, PartitionService partitionService) { + this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; this.partitionService = partitionService; @@ -134,7 +140,8 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); transportNotificationsConsumer = queueProvider.createTransportNotificationsConsumer(); - transportNotificationsConsumer.subscribe(); + TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceInfoProvider.getServiceId()); + transportNotificationsConsumer.subscribe(Collections.singleton(tpi)); transportApiRequestTemplate.init(); mainConsumerExecutor.execute(() -> { while (!stopped) { @@ -145,10 +152,7 @@ public class DefaultTransportService implements TransportService { } records.forEach(record -> { try { - ToTransportMsg toTransportMsg = record.getValue(); - if (toTransportMsg.hasToDeviceSessionMsg()) { - processToTransportMsg(toTransportMsg.getToDeviceSessionMsg()); - } + processToTransportMsg(record.getValue()); } catch (Throwable e) { log.warn("Failed to process the notification.", e); } @@ -195,7 +199,7 @@ public class DefaultTransportService implements TransportService { @Override public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) { - sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); + sessions.putIfAbsent(toSessionId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener)); } @Override @@ -210,22 +214,6 @@ public class DefaultTransportService implements TransportService { } } - public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { - - TransportProtos.GetTenantRoutingInfoRequestMsg msg = TransportProtos.GetTenantRoutingInfoRequestMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .build(); - TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetTenantRoutingInfoRequestMsg(msg).build()); - try { - TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); - TransportProtos.GetTenantRoutingInfoResponseMsg routingInfo = response.getValue().getGetTenantRoutingInfoResponseMsg(); - return new TenantRoutingInfo(tenantId, routingInfo.getIsolatedTbCore(), routingInfo.getIsolatedTbRuleEngine()); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - @Override public void process(TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { log.trace("Processing msg: {}", msg); @@ -253,7 +241,7 @@ public class DefaultTransportService implements TransportService { @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setSubscriptionInfo(msg).build(), callback); @@ -340,13 +328,51 @@ public class DefaultTransportService implements TransportService { } } - //TODO 2.5: Need to handle timeouts on the transport level and not on the Device Actor Level. + private void processTimeout(String requestId) { + RpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + SessionMetaData md = sessions.get(data.getSessionId()); + if (md != null) { + SessionMsgListener listener = md.getListener(); + transportCallbackExecutor.submit(() -> { + TransportProtos.ToServerRpcResponseMsg responseMsg = + TransportProtos.ToServerRpcResponseMsg.newBuilder() + .setRequestId(data.getRequestId()) + .setError("timeout").build(); + listener.onToServerRpcResponse(responseMsg); + }); + if (md.getSessionType() == TransportProtos.SessionType.SYNC) { + deregisterSession(md.getSessionInfo()); + } + } else { + log.debug("[{}] Missing session.", data.getSessionId()); + } + } + } + @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { reportActivityInternal(sessionInfo); -// sendToRuleEngine(sessionInfo, TransportToRuleEngineMsg.newBuilder().setSessionInfo(sessionInfo). -// setToServerRPCCallRequest(msg).build(), new TransportTbQueueCallback(callback)); + UUID sessionId = toSessionId(sessionInfo); + TenantId tenantId = getTenantId(sessionInfo); + DeviceId deviceId = getDeviceId(sessionInfo); + JsonObject json = new JsonObject(); + json.addProperty("method", msg.getMethodName()); + json.add("params", JsonUtils.parse(msg.getParams())); + + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("deviceName", sessionInfo.getDeviceName()); + metaData.putValue("deviceType", sessionInfo.getDeviceType()); + metaData.putValue("requestId", Integer.toString(msg.getRequestId())); + metaData.putValue("serviceId", serviceInfoProvider.getServiceId()); + metaData.putValue("sessionId", sessionId.toString()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, metaData, TbMsgDataType.JSON, gson.toJson(json)); + sendToRuleEngine(tenantId, tbMsg, new TransportTbQueueCallback(callback)); + + String requestId = sessionId + "-" + msg.getRequestId(); + toServerRpcPendingMap.put(requestId, new RpcRequestMetadata(sessionId, msg.getRequestId())); + schedulerExecutor.schedule(() -> processTimeout(requestId), clientSideRpcTimeout, TimeUnit.MILLISECONDS); } } @@ -364,7 +390,7 @@ public class DefaultTransportService implements TransportService { } private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { - UUID sessionId = toId(sessionInfo); + UUID sessionId = toSessionId(sessionInfo); SessionMetaData sessionMetaData = sessions.get(sessionId); if (sessionMetaData != null) { sessionMetaData.updateLastActivityTime(); @@ -377,7 +403,7 @@ public class DefaultTransportService implements TransportService { sessions.forEach((uuid, sessionMD) -> { if (sessionMD.getLastActivityTime() < expTime) { if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); } process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); sessions.remove(uuid); @@ -407,7 +433,7 @@ public class DefaultTransportService implements TransportService { @Override public void registerSyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout) { SessionMetaData currentSession = new SessionMetaData(sessionInfo, TransportProtos.SessionType.SYNC, listener); - sessions.putIfAbsent(toId(sessionInfo), currentSession); + sessions.putIfAbsent(toSessionId(sessionInfo), currentSession); ScheduledFuture executorFuture = schedulerExecutor.schedule(() -> { listener.onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); @@ -419,18 +445,18 @@ public class DefaultTransportService implements TransportService { @Override public void deregisterSession(TransportProtos.SessionInfoProto sessionInfo) { - SessionMetaData currentSession = sessions.get(toId(sessionInfo)); + SessionMetaData currentSession = sessions.get(toSessionId(sessionInfo)); if (currentSession != null && currentSession.hasScheduledFuture()) { log.debug("Stopping scheduler to avoid resending response if request has been ack."); currentSession.getScheduledFuture().cancel(false); } - sessions.remove(toId(sessionInfo)); + sessions.remove(toSessionId(sessionInfo)); } @Override public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { if (log.isTraceEnabled()) { - log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg); + log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } if (!rateLimitEnabled) { return true; @@ -442,7 +468,7 @@ public class DefaultTransportService implements TransportService { callback.onError(new TbRateLimitsException(EntityType.TENANT)); } if (log.isTraceEnabled()) { - log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg); + log.trace("[{}][{}] Tenant level rate limit detected: {}", toSessionId(sessionInfo), tenantId, msg); } return false; } @@ -453,7 +479,7 @@ public class DefaultTransportService implements TransportService { callback.onError(new TbRateLimitsException(EntityType.DEVICE)); } if (log.isTraceEnabled()) { - log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg); + log.trace("[{}][{}] Device level rate limit detected: {}", toSessionId(sessionInfo), deviceId, msg); } return false; } @@ -461,7 +487,7 @@ public class DefaultTransportService implements TransportService { return true; } - protected void processToTransportMsg(TransportProtos.DeviceActorToTransportMsg toSessionMsg) { + protected void processToTransportMsg(TransportProtos.ToTransportMsg toSessionMsg) { UUID sessionId = new UUID(toSessionMsg.getSessionIdMSB(), toSessionMsg.getSessionIdLSB()); SessionMetaData md = sessions.get(sessionId); if (md != null) { @@ -480,6 +506,8 @@ public class DefaultTransportService implements TransportService { listener.onToDeviceRpcRequest(toSessionMsg.getToDeviceRequest()); } if (toSessionMsg.hasToServerResponse()) { + String requestId = sessionId + "-" + toSessionMsg.getToServerResponse().getRequestId(); + toServerRpcPendingMap.remove(requestId); listener.onToServerRpcResponse(toSessionMsg.getToServerResponse()); } }); @@ -492,7 +520,7 @@ public class DefaultTransportService implements TransportService { } } - protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) { + protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java similarity index 76% rename from common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java index d06046fd71..2c5da0d3ea 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/core/ToServerRpcResponseMsg.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/RpcRequestMetadata.java @@ -13,20 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.msg.core; +package org.thingsboard.server.common.transport.service; import lombok.Data; -/** - * @author Andrew Shvayka - */ -@Data -public class ToServerRpcResponseMsg { +import java.util.UUID; +@Data +public class RpcRequestMetadata { + private final UUID sessionId; private final int requestId; - private final String data; - - public boolean isSuccess() { - return true; - } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index 5b76ecf624..baac594a17 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.api; import org.thingsboard.server.common.data.id.DeviceId; + +import java.util.UUID; import java.util.function.Consumer; /** @@ -23,7 +25,7 @@ import java.util.function.Consumer; */ public interface RuleEngineRpcService { - void sendRpcReplyToDevice(DeviceId deviceId, int requestId, String body); + void sendRpcReplyToDevice(String serviceId, UUID sessionId, int requestId, String body); void sendRpcRequestToDevice(RuleEngineDeviceRpcRequest request, Consumer consumer); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 393ef0bd7c..145e73d450 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; +import java.util.UUID; + @Slf4j @RuleNode( type = ComponentType.ACTION, @@ -50,15 +52,21 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { + String serviceIdStr = msg.getMetaData().getValue(config.getServiceIdMetaDataAttribute()); + String sessionIdStr = msg.getMetaData().getValue(config.getSessionIdMetaDataAttribute()); String requestIdStr = msg.getMetaData().getValue(config.getRequestIdMetaDataAttribute()); if (msg.getOriginator().getEntityType() != EntityType.DEVICE) { ctx.tellFailure(msg, new RuntimeException("Message originator is not a device entity!")); } else if (StringUtils.isEmpty(requestIdStr)) { ctx.tellFailure(msg, new RuntimeException("Request id is not present in the metadata!")); + } else if (StringUtils.isEmpty(serviceIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Service id is not present in the metadata!")); + } else if (StringUtils.isEmpty(sessionIdStr)) { + ctx.tellFailure(msg, new RuntimeException("Session id is not present in the metadata!")); } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReplyToDevice(new DeviceId(msg.getOriginator().getId()), Integer.parseInt(requestIdStr), msg.getData()); + ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); ctx.tellSuccess(msg); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java index d01f28f136..bd3a3cdfb2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRpcReplyNodeConfiguration.java @@ -16,18 +16,40 @@ package org.thingsboard.rule.engine.rpc; import lombok.Data; +import org.springframework.util.StringUtils; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.DataConstants; @Data public class TbSendRpcReplyNodeConfiguration implements NodeConfiguration { + public static final String SERVICE_ID = "serviceId"; + public static final String SESSION_ID = "sessionId"; + public static final String REQUEST_ID = "requestId"; + + private String serviceIdMetaDataAttribute; + private String sessionIdMetaDataAttribute; private String requestIdMetaDataAttribute; @Override public TbSendRpcReplyNodeConfiguration defaultConfiguration() { TbSendRpcReplyNodeConfiguration configuration = new TbSendRpcReplyNodeConfiguration(); - configuration.setRequestIdMetaDataAttribute("requestId"); + configuration.setServiceIdMetaDataAttribute(SERVICE_ID); + configuration.setSessionIdMetaDataAttribute(SESSION_ID); + configuration.setRequestIdMetaDataAttribute(REQUEST_ID); return configuration; } + + public String getServiceIdMetaDataAttribute() { + return !StringUtils.isEmpty(serviceIdMetaDataAttribute) ? serviceIdMetaDataAttribute : SERVICE_ID; + } + + public String getSessionIdMetaDataAttribute() { + return !StringUtils.isEmpty(sessionIdMetaDataAttribute) ? sessionIdMetaDataAttribute : SESSION_ID; + } + + public String getRequestIdMetaDataAttribute() { + return !StringUtils.isEmpty(requestIdMetaDataAttribute) ? requestIdMetaDataAttribute : REQUEST_ID; + } } + diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 70d1ff8fbe..b94148f608 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -81,15 +81,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -98,9 +98,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -110,7 +110,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -118,7 +118,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -133,7 +133,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -149,7 +149,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 5ddb7c0553..0fa2788cac 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -82,15 +82,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -99,9 +99,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -111,7 +111,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -119,7 +119,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -134,7 +134,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -150,7 +150,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index c421c43e33..747427caf8 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -17,10 +17,20 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # MQTT server parameters transport: @@ -102,15 +112,15 @@ queue: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}" - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}" + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - topic: "${TB_QUEUE_CORE_TOPIC:tb.core}" + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -119,9 +129,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout @@ -131,7 +141,7 @@ queue: # JS response auto commit interval response_auto_commit_interval: "${REMOTE_JS_RESPONSE_AUTO_COMMIT_INTERVAL_MS:100}" rule-engine: - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb.rule-engine}" + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: @@ -139,7 +149,7 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - name: "Main" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb.rule-engine.main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -154,7 +164,7 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb.rule-engine.hp}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" @@ -170,7 +180,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}" + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" service: From e83d69c40ced610b0cfc7db05424b8b4bb43e73f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Apr 2020 20:16:51 +0300 Subject: [PATCH 065/196] isolated tenant improvements --- .../server/controller/TenantController.java | 11 +++++++++++ .../service/install/SqlDatabaseUpgradeService.java | 5 ++++- ui/src/app/tenant/tenant-fieldset.tpl.html | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 9def943e88..e9a793fb34 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -28,6 +28,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; @@ -72,6 +73,16 @@ public class TenantController extends BaseController { try { boolean newTenant = tenant.getId() == null; + if (!newTenant) { + Tenant oldTenant = tenantService.findTenantById(tenant.getTenantId()); + if (oldTenant.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { + throw new ThingsboardException("Field isolatedTbCore from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + if (oldTenant.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { + throw new ThingsboardException("Field isolatedTbRuleEngine from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + } + Operation operation = newTenant ? Operation.CREATE : Operation.WRITE; accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index bcbd77ef28..f673f498fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,7 +221,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } - conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + try { + conn.createStatement().execute("ALTER TABLE tenant ADD COLUMN isolated_tb_core boolean DEFAULT (false), ADD COLUMN isolated_tb_rule_engine boolean DEFAULT (false)"); + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/ui/src/app/tenant/tenant-fieldset.tpl.html b/ui/src/app/tenant/tenant-fieldset.tpl.html index 35b3dee289..b75ea30f06 100644 --- a/ui/src/app/tenant/tenant-fieldset.tpl.html +++ b/ui/src/app/tenant/tenant-fieldset.tpl.html @@ -47,14 +47,14 @@ - {{'tenant.isolated-tb-core' | translate}}
{{'tenant.isolated-tb-core-details' | translate}} - {{'tenant.isolated-tb-rule-engine' | translate}}
{{'tenant.isolated-tb-rule-engine-details' | translate}} From d024ba56e89fad42842a5c9b9aafacb865d6466e Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Apr 2020 21:33:07 +0300 Subject: [PATCH 066/196] removed tenant validation from controller --- .../server/controller/TenantController.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index e9a793fb34..623fbb65c9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -73,16 +73,6 @@ public class TenantController extends BaseController { try { boolean newTenant = tenant.getId() == null; - if (!newTenant) { - Tenant oldTenant = tenantService.findTenantById(tenant.getTenantId()); - if (oldTenant.isIsolatedTbCore() != tenant.isIsolatedTbCore()) { - throw new ThingsboardException("Field isolatedTbCore from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - if (oldTenant.isIsolatedTbRuleEngine() != tenant.isIsolatedTbRuleEngine()) { - throw new ThingsboardException("Field isolatedTbRuleEngine from Tenant can't be changed.", ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - } - Operation operation = newTenant ? Operation.CREATE : Operation.WRITE; accessControlService.checkPermission(getCurrentUser(), Resource.TENANT, operation, From 2e122d86263aba1160b806d3229533f28e322392 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Apr 2020 21:34:32 +0300 Subject: [PATCH 067/196] refactored --- .../java/org/thingsboard/server/controller/TenantController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 623fbb65c9..9def943e88 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -28,7 +28,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; From 63bac58ae84d31f3976ab447c088741fbfc50aeb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 08:27:44 +0300 Subject: [PATCH 068/196] Implementation of statistics counters --- .../server/actors/ActorSystemContext.java | 4 - .../actors/ruleChain/DefaultTbContext.java | 11 +- .../RuleChainActorMessageProcessor.java | 10 +- .../actors/service/DefaultActorService.java | 92 ------------- .../server/controller/BaseController.java | 3 +- .../server/controller/DeviceController.java | 6 +- .../controller/TelemetryController.java | 8 +- .../queue/DefaultTbClusterService.java | 113 ++++++++++++---- .../queue/DefaultTbCoreConsumerService.java | 24 ++-- .../service/queue/TbClusterService.java | 22 ++- .../service/queue/TbCoreConsumerStats.java | 25 +++- .../rpc/DefaultTbCoreDeviceRpcService.java | 5 +- .../rpc/DefaultTbRuleEngineRpcService.java | 7 +- .../state/DefaultDeviceStateService.java | 21 +-- .../DefaultSubscriptionManagerService.java | 5 +- .../DefaultTbLocalSubscriptionService.java | 12 +- .../DefaultTelemetrySubscriptionService.java | 23 ++-- .../TbTransportQueueProducerProvider.java | 2 - .../rule/engine/api/TbContext.java | 2 +- .../rule/engine/action/TbMsgCountNode.java | 1 - .../rule/engine/rest/TbHttpClient.java | 23 +--- .../engine/rest/TbRedisQueueProcessor.java | 126 ------------------ .../rule/engine/rest/TbRestApiCallNode.java | 19 +-- 23 files changed, 186 insertions(+), 378 deletions(-) delete mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7822438724..fb4ff77359 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -159,10 +159,6 @@ public class ActorSystemContext { @Getter private TbClusterService clusterService; - @Autowired - @Getter - private TbQueueProducerProvider producerProvider; - @Autowired @Getter private TimeseriesService tsService; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 0e8033fac2..13f7b1122d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -17,10 +17,8 @@ package org.thingsboard.server.actors.ruleChain; import akka.actor.ActorRef; import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; @@ -36,7 +34,6 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; @@ -45,8 +42,8 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; @@ -62,11 +59,9 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import scala.concurrent.duration.Duration; @@ -136,7 +131,7 @@ class DefaultTbContext implements TbContext { .setTenantIdMSB(getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(getTenantId().getId().getLeastSignificantBits()) .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), new SimpleTbQueueCallback(onSuccess, onFailure)); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, new SimpleTbQueueCallback(onSuccess, onFailure)); } @Override @@ -193,7 +188,7 @@ class DefaultTbContext implements TbContext { if (failureMessage != null) { msg.setFailureMessage(failureMessage); } - mainCtx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg.build()), new SimpleTbQueueCallback(onSuccess, onFailure)); + mainCtx.getClusterService().pushMsgToRuleEngine(tpi, tbMsg.getId(), msg.build(), new SimpleTbQueueCallback(onSuccess, onFailure)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index dd0a515a33..a374e590b5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -44,10 +44,9 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; +import org.thingsboard.server.service.queue.TbClusterService; import java.util.ArrayList; import java.util.Collections; @@ -68,7 +67,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor nodeActors; private final Map> nodeRoutes; private final RuleChainService service; - private final TbQueueProducer> producer; + private final TbClusterService clusterService; private String ruleChainName; private RuleNodeId firstId; @@ -84,7 +83,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(); this.nodeRoutes = new HashMap<>(); this.service = systemContext.getRuleChainService(); - this.producer = systemContext.getProducerProvider().getRuleEngineMsgProducer(); + this.clusterService = systemContext.getClusterService(); } @Override @@ -255,7 +254,6 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor(newMsg.getId(), toQueueMsg), callbackWrapper); + clusterService.pushMsgToRuleEngine(tpi, newMsg.getId(), toQueueMsg, callbackWrapper); } private boolean contains(Set relationTypes, String type) { diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 537e897583..7b04e82786 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -97,96 +97,4 @@ public class DefaultActorService implements ActorService { } } - @Value("${cluster.stats.enabled:false}") - private boolean statsEnabled; - - //TODO 2.5 - private final AtomicInteger sentClusterMsgs = new AtomicInteger(0); - private final AtomicInteger receivedClusterMsgs = new AtomicInteger(0); - - - @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") - public void printStats() { - if (statsEnabled) { - int sent = sentClusterMsgs.getAndSet(0); - int received = receivedClusterMsgs.getAndSet(0); - if (sent > 0 || received > 0) { - log.info("Cluster msgs sent [{}] received [{}]", sent, received); - } - } - } - - //TODO 2.5 -// @Override -// public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) { -// if (statsEnabled) { -// receivedClusterMsgs.incrementAndGet(); -// } -// ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType()); -// if (log.isDebugEnabled()) { -// log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress); -// log.info("MSG: {}", msg); -// } -// switch (msg.getMessageType()) { -// case CLUSTER_ACTOR_MESSAGE: -// java.util.Optional decodedMsg = actorContext.getEncodingService() -// .decode(msg.getPayload().toByteArray()); -// if (decodedMsg.isPresent()) { -// appActor.tell(decodedMsg.get(), ActorRef.noSender()); -// } else { -// log.error("Error during decoding cluster proto message"); -// } -// break; -// case TO_ALL_NODES_MSG: -// //TODO -// break; -// case CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE: -// actorContext.getTsSubService().onNewRemoteSubscription(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_SUBSCRIPTION_UPDATE_MESSAGE: -// actorContext.getTsSubService().onRemoteSubscriptionUpdate(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_SUBSCRIPTION_CLOSE_MESSAGE: -// actorContext.getTsSubService().onRemoteSubscriptionClose(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_SESSION_CLOSE_MESSAGE: -// actorContext.getTsSubService().onRemoteSessionClose(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_ATTR_UPDATE_MESSAGE: -// actorContext.getTsSubService().onRemoteAttributesUpdate(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TELEMETRY_TS_UPDATE_MESSAGE: -// actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE: -// actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE: -// actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray()); -// break; -// case CLUSTER_TRANSACTION_SERVICE_MESSAGE: -// actorContext.getRuleChainTransactionService().onRemoteTransactionMsg(serverAddress, msg.getPayload().toByteArray()); -// break; -// } -// } -// @Override -// public void onSendMsg(ClusterAPIProtos.ClusterMessage msg) { -// if (statsEnabled) { -// sentClusterMsgs.incrementAndGet(); -// } -// rpcManagerActor.tell(msg, ActorRef.noSender()); -// } -// -// @Override -// public void onRpcSessionCreateRequestMsg(RpcSessionCreateRequestMsg msg) { -// if (statsEnabled) { -// sentClusterMsgs.incrementAndGet(); -// } -// rpcManagerActor.tell(msg, ActorRef.noSender()); -// } -// @Override -// public void onBroadcastMsg(RpcBroadcastMsg msg) { -// rpcManagerActor.tell(msg, ActorRef.noSender()); -// } - } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index fbccaedb9e..73b25d4de2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -668,7 +667,7 @@ public abstract class BaseController { } } TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - tbClusterService.onToRuleEngineMsg(user.getTenantId(), entityId, tbMsg); + tbClusterService.pushMsgToRuleEngine(user.getTenantId(), entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index afa394b98c..ca61ecc8e9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -99,8 +99,8 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - tbClusterService.onToCoreMsg(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), - savedDevice.getId(), savedDevice.getName(), savedDevice.getType())); + tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), + savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); logEntityAction(savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), @@ -254,7 +254,7 @@ public class DeviceController extends BaseController { Device device = checkDeviceId(deviceCredentials.getDeviceId(), Operation.WRITE_CREDENTIALS); DeviceCredentials result = checkNotNull(deviceCredentialsService.updateDeviceCredentials(getCurrentUser().getTenantId(), deviceCredentials)); - tbClusterService.onToCoreMsg(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId())); + tbClusterService.pushMsgToCore(new DeviceCredentialsUpdateNotificationMsg(getCurrentUser().getTenantId(), deviceCredentials.getDeviceId()), null); logEntityAction(device.getId(), device, device.getCustomerId(), diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index d0c4be2386..8352e3f457 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -364,8 +364,8 @@ public class TelemetryController extends BaseController { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); - tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, keysToNotify)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, keysToNotify), null); } result.setResult(new ResponseEntity<>(HttpStatus.OK)); } @@ -399,8 +399,8 @@ public class TelemetryController extends BaseController { logAttributesUpdated(user, entityId, scope, attributes, null); if (entityId.getEntityType() == EntityType.DEVICE) { DeviceId deviceId = new DeviceId(entityId.getId()); - tbClusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes)); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( + user.getTenantId(), deviceId, scope, attributes), null); } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f93657427d..f0054771d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.data.EntityType; @@ -27,7 +29,13 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos.*; +import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,12 +46,22 @@ import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; import java.util.HashSet; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; @Service @Slf4j public class DefaultTbClusterService implements TbClusterService { - protected TbQueueProducerProvider producerProvider; + @Value("${cluster.stats.enabled:false}") + private boolean statsEnabled; + + private final AtomicInteger toCoreMsgs = new AtomicInteger(0); + private final AtomicInteger toCoreNfs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineMsgs = new AtomicInteger(0); + private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0); + private final AtomicInteger toTransportNfs = new AtomicInteger(0); + + private final TbQueueProducerProvider producerProvider; private final PartitionService partitionService; private final DataDecodingEncodingService encodingService; @@ -54,33 +72,29 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onToRuleEngineMsg(TenantId tenantId, EntityId entityId, TbMsg tbMsg) { - if (tenantId.isNullUid()) { - if (entityId.getEntityType().equals(EntityType.TENANT)) { - tenantId = new TenantId(entityId.getId()); - } else { - log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); - return; - } - } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + + @Override + public void pushMsgToCore(TopicPartitionInfo tpi, UUID msgId, ToCoreMsg msg, TbQueueCallback callback) { + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toCoreMsgs.incrementAndGet(); } @Override - public void onToCoreMsg(ToDeviceActorNotificationMsg msg) { + public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); byte[] msgBytes = encodingService.encode(msg); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), null); + producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); + toCoreMsgs.incrementAndGet(); } @Override - public void onToCoreMsg(String serviceId, FromDeviceRpcResponse response) { + public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) @@ -88,11 +102,37 @@ public class DefaultTbClusterService implements TbClusterService { .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); response.getResponse().ifPresent(builder::setResponse); ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toCoreNfs.incrementAndGet(); + } + + @Override + public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); } @Override - public void onToRuleEngineMsg(String serviceId, FromDeviceRpcResponse response) { + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg tbMsg, TbQueueCallback callback) { + if (tenantId.isNullUid()) { + if (entityId.getEntityType().equals(EntityType.TENANT)) { + tenantId = new TenantId(entityId.getId()); + } else { + log.warn("[{}][{}] Received invalid message: {}", tenantId, entityId, tbMsg); + return; + } + } + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); + toRuleEngineMsgs.incrementAndGet(); + } + + @Override + public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) @@ -100,13 +140,15 @@ public class DefaultTbClusterService implements TbClusterService { .setError(response.getError().isPresent() ? response.getError().get().ordinal() : -1); response.getResponse().ifPresent(builder::setResponse); ToRuleEngineNotificationMsg msg = ToRuleEngineNotificationMsg.newBuilder().setFromDeviceRpcResponse(builder).build(); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), null); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(response.getId(), msg), callback); + toRuleEngineNfs.incrementAndGet(); } @Override - public void onToTransportMsg(String serviceId, ToTransportMsg response) { + public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); - producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), null); + producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); + toTransportNfs.incrementAndGet(); } @Override @@ -120,12 +162,13 @@ public class DefaultTbClusterService implements TbClusterService { TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)) { - TbQueueProducer> toCoreProducer = producerProvider.getTbCoreNotificationsMsgProducer(); + TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); + toCoreNfs.incrementAndGet(); } // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); @@ -134,6 +177,22 @@ public class DefaultTbClusterService implements TbClusterService { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + } + + @Scheduled(fixedDelayString = "${cluster.stats.print_interval_ms}") + public void printStats() { + if (statsEnabled) { + int toCoreMsgCnt = toCoreMsgs.getAndSet(0); + int toCoreNfsCnt = toCoreNfs.getAndSet(0); + int toRuleEngineMsgsCnt = toRuleEngineMsgs.getAndSet(0); + int toRuleEngineNfsCnt = toRuleEngineNfs.getAndSet(0); + int toTransportNfsCnt = toTransportNfs.getAndSet(0); + if (toCoreMsgCnt > 0 || toCoreNfsCnt > 0 || toRuleEngineMsgsCnt > 0 || toRuleEngineNfsCnt > 0 || toTransportNfsCnt > 0) { + log.info("To TbCore: [{}] messages [{}] notifications; To TbRuleEngine: [{}] messages [{}] notifications; To Transport: [{}] notifications", + toCoreMsgCnt, toCoreNfsCnt, toRuleEngineMsgsCnt, toRuleEngineNfsCnt, toTransportNfsCnt); + } } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 263e6eab24..c943e4c4d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -183,21 +183,24 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbCallback callback) { - ToCoreNotificationMsg toCoreMsg = msg.getValue(); - if (toCoreMsg.hasToLocalSubscriptionServiceMsg()) { - log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreMsg.getToLocalSubscriptionServiceMsg()); - forwardToLocalSubMgrService(toCoreMsg.getToLocalSubscriptionServiceMsg(), callback); - } else if (toCoreMsg.hasFromDeviceRpcResponse()) { - log.trace("[{}] Forwarding message to RPC service {}", id, toCoreMsg.getFromDeviceRpcResponse()); - forwardToCoreRpcService(toCoreMsg.getFromDeviceRpcResponse(), callback); - } else if (toCoreMsg.getComponentLifecycleMsg() != null && !toCoreMsg.getComponentLifecycleMsg().isEmpty()) { - Optional actorMsg = encodingService.decode(toCoreMsg.getComponentLifecycleMsg().toByteArray()); + ToCoreNotificationMsg toCoreNotification = msg.getValue(); + if (toCoreNotification.hasToLocalSubscriptionServiceMsg()) { + log.trace("[{}] Forwarding message to local subscription service {}", id, toCoreNotification.getToLocalSubscriptionServiceMsg()); + forwardToLocalSubMgrService(toCoreNotification.getToLocalSubscriptionServiceMsg(), callback); + } else if (toCoreNotification.hasFromDeviceRpcResponse()) { + log.trace("[{}] Forwarding message to RPC service {}", id, toCoreNotification.getFromDeviceRpcResponse()); + forwardToCoreRpcService(toCoreNotification.getFromDeviceRpcResponse(), callback); + } else if (toCoreNotification.getComponentLifecycleMsg() != null && !toCoreNotification.getComponentLifecycleMsg().isEmpty()) { + Optional actorMsg = encodingService.decode(toCoreNotification.getComponentLifecycleMsg().toByteArray()); if (actorMsg.isPresent()) { log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get()); actorContext.tell(actorMsg.get(), ActorRef.noSender()); } callback.onSuccess(); } + if (statsEnabled) { + stats.log(toCoreNotification); + } } private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { @@ -246,6 +249,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService 0) { - log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]", + log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + " deviceState [{}] subMgr [{}] coreNfs [{}]", total, sessionEventCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), - subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); + subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0) + , deviceStateCounter.getAndSet(0), subscriptionMsgCounter.getAndSet(0), toCoreNotificationsCounter.getAndSet(0)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index 6d70056af0..8a1423f97c 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.rpc; import akka.actor.ActorRef; -import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -147,7 +146,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); } } else { - clusterService.onToRuleEngineMsg(originServiceId, response); + clusterService.pushNotificationToRuleEngine(originServiceId, response, null); } } @@ -170,7 +169,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { try { TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - clusterService.onToRuleEngineMsg(msg.getTenantId(), msg.getDeviceId(), tbMsg); + clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null); } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java index 909690ff6c..0ec730b7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java @@ -22,7 +22,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.RpcError; import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest; import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -96,7 +95,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setToServerResponse(responseMsg) .build(); - clusterService.onToTransportMsg(serviceId, msg); + clusterService.pushNotificationToTransport(serviceId, msg, null); } @Override @@ -148,7 +147,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi } } else { log.trace("[{}] Forwarding msg {} to queue actor!", msg.getDeviceId(), msg); - clusterService.onToCoreMsg(rpcMsg); + clusterService.pushMsgToCore(rpcMsg, null); } } @@ -160,7 +159,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi log.warn("Failed to find tbCoreRpcService for local service. Possible duplication of serviceIds."); } } else { - clusterService.onToCoreMsg(originServiceId, response); + clusterService.pushNotificationToCore(originServiceId, response, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 87f46f2072..1e3ab73a26 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -29,7 +29,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -56,8 +55,8 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -106,7 +105,7 @@ public class DefaultDeviceStateService implements DeviceStateService { private final DeviceService deviceService; private final AttributesService attributesService; private final TimeseriesService tsService; - private final TbQueueProducerProvider producerProvider; + private final TbClusterService clusterService; private final PartitionService partitionService; private TelemetrySubscriptionService tsSubService; @@ -137,12 +136,12 @@ public class DefaultDeviceStateService implements DeviceStateService { public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, - TbQueueProducerProvider producerProvider, PartitionService partitionService) { + TbClusterService clusterService, PartitionService partitionService) { this.tenantService = tenantService; this.deviceService = deviceService; this.attributesService = attributesService; this.tsService = tsService; - this.producerProvider = producerProvider; + this.clusterService = clusterService; this.partitionService = partitionService; } @@ -413,8 +412,6 @@ public class DefaultDeviceStateService implements DeviceStateService { } private void sendDeviceEvent(TenantId tenantId, DeviceId deviceId, boolean added, boolean updated, boolean deleted) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - log.trace("[{}][{}] Device is monitored on partition: {}", tenantId, deviceId, tpi); TransportProtos.DeviceStateServiceMsgProto.Builder builder = TransportProtos.DeviceStateServiceMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -424,8 +421,7 @@ public class DefaultDeviceStateService implements DeviceStateService { builder.setUpdated(updated); builder.setDeleted(deleted); TransportProtos.DeviceStateServiceMsgProto msg = builder.build(); - producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), - TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build()), null); + clusterService.pushMsgToCore(tenantId, deviceId, TransportProtos.ToCoreMsg.newBuilder().setDeviceStateServiceMsg(msg).build(), null); } private void onDeviceDeleted(TenantId tenantId, DeviceId deviceId) { @@ -497,12 +493,7 @@ public class DefaultDeviceStateService implements DeviceStateService { try { TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON , json.writeValueAsString(state)); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, stateData.getTenantId(), stateData.getDeviceId()); - TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() - .setTenantIdMSB(stateData.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(stateData.getTenantId().getId().getLeastSignificantBits()) - .setTbMsg(TbMsg.toByteString(tbMsg)).build(); - producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), null); + clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index e841acae12..67c81ac06f 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -245,8 +245,9 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } } } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope)) { - clusterService.onToCoreMsg(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))); + clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, + new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) + , null); } } callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 12c5e206e3..d57067fa28 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -28,16 +28,14 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate; @@ -70,19 +68,17 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer private PartitionService partitionService; @Autowired - private TbQueueProducerProvider producerProvider; + private TbClusterService clusterService; @Autowired @Lazy private SubscriptionManagerService subscriptionManagerService; private ExecutorService wsCallBackExecutor; - private TbQueueProducer> toCoreProducer; @PostConstruct public void initExecutor() { wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-sub-callback")); - toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy @@ -140,7 +136,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } else { // Push to the queue; TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); } } @@ -181,7 +177,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } else { // Push to the queue; TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null); } } else { log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index b3c728a25b..665a787e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,17 +32,15 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -69,22 +67,20 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio private final AttributesService attrService; private final TimeseriesService tsService; - private final TbQueueProducerProvider producerProvider; + private final TbClusterService clusterService; private final PartitionService partitionService; private Optional subscriptionManagerService; - private TbQueueProducer> toCoreProducer; - private ExecutorService tsCallBackExecutor; private ExecutorService wsCallBackExecutor; public DefaultTelemetrySubscriptionService(AttributesService attrService, TimeseriesService tsService, - TbQueueProducerProvider producerProvider, + TbClusterService clusterService, PartitionService partitionService) { this.attrService = attrService; this.tsService = tsService; - this.producerProvider = producerProvider; + this.clusterService = clusterService; this.partitionService = partitionService; } @@ -97,7 +93,6 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio public void initExecutor() { tsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ts-callback")); wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ts-service-ws-callback")); - toCoreProducer = producerProvider.getTbCoreMsgProducer(); } @PreDestroy @@ -172,7 +167,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } @@ -186,7 +181,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio } } else { TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - toCoreProducer.send(tpi, new TbProtoQueueMsg<>(entityId.getId(), toCoreMsg), null); + clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 0ebd59ec24..bced3f83ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -23,13 +23,11 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import javax.annotation.PostConstruct; -//TODO 2.5 Maybe remove this service if it is not used. @Service @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TbTransportQueueProducerProvider implements TbQueueProducerProvider { private final TbTransportQueueFactory tbQueueProvider; - private TbQueueProducer> toTransport; private TbQueueProducer> toRuleEngine; private TbQueueProducer> toTbCore; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 4299aeb867..daab510102 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -211,7 +211,7 @@ public interface TbContext { ResultSetFuture submitCassandraTask(CassandraStatementTask task); - //TODO 2.5: - need to remove this. + @Deprecated RedisTemplate getRedisTemplate(); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java index 10e664bc16..19d8515455 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java @@ -75,7 +75,6 @@ public class TbMsgCountNode implements TbNode { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay)); - //TODO 2.5: Callback? TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, gson.toJson(telemetryJson)); ctx.enqueueForTellNext(tbMsg, SUCCESS); scheduleTickMsg(ctx); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 5b30f4792c..ba860c103b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.Netty4ClientHttpRequestFactory; import org.springframework.util.concurrent.ListenableFuture; @@ -84,7 +83,7 @@ class TbHttpClient { } } - void processMessage(TbContext ctx, TbMsg msg, TbRedisQueueProcessor queueProcessor) { + void processMessage(TbContext ctx, TbMsg msg) { String endpointUrl = TbNodeUtils.processPattern(config.getRestEndpointUrlPattern(), msg.getMetaData()); HttpHeaders headers = prepareHeaders(msg.getMetaData()); HttpMethod method = HttpMethod.valueOf(config.getRequestMethod()); @@ -95,13 +94,6 @@ class TbHttpClient { future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { - if (config.isUseRedisQueueForMsgPersistence()) { - if (throwable instanceof HttpClientErrorException) { - processHttpClientError(((HttpClientErrorException) throwable).getStatusCode(), msg, queueProcessor); - } else { - queueProcessor.pushOnFailure(msg); - } - } TbMsg next = processException(ctx, msg, throwable); ctx.tellFailure(next, throwable); } @@ -109,15 +101,9 @@ class TbHttpClient { @Override public void onSuccess(ResponseEntity responseEntity) { if (responseEntity.getStatusCode().is2xxSuccessful()) { - if (config.isUseRedisQueueForMsgPersistence()) { - queueProcessor.resetCounter(); - } TbMsg next = processResponse(ctx, msg, responseEntity); ctx.tellSuccess(next); } else { - if (config.isUseRedisQueueForMsgPersistence()) { - processHttpClientError(responseEntity.getStatusCode(), msg, queueProcessor); - } TbMsg next = processFailureResponse(ctx, msg, responseEntity); ctx.tellNext(next, TbRelationTypes.FAILURE); } @@ -183,11 +169,4 @@ class TbHttpClient { } } - private void processHttpClientError(HttpStatus statusCode, TbMsg msg, TbRedisQueueProcessor queueProcessor) { - if (statusCode.is4xxClientError()) { - log.warn("[{}] Client error during message delivering!", msg); - } else { - queueProcessor.pushOnFailure(msg); - } - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java deleted file mode 100644 index 015cced6c6..0000000000 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRedisQueueProcessor.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.rest; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.redis.core.ListOperations; -import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Data -@Slf4j -class TbRedisQueueProcessor { - - private static final int MAX_QUEUE_SIZE = Integer.MAX_VALUE; - - private final TbContext ctx; - private final TbHttpClient httpClient; - private final ExecutorService executor; - private final ListOperations listOperations; - private final String redisKey; - private final boolean trimQueue; - private final int maxQueueSize; - - private AtomicInteger failuresCounter; - private Future future; - - TbRedisQueueProcessor(TbContext ctx, TbHttpClient httpClient, boolean trimQueue, int maxQueueSize) { - this.ctx = ctx; - this.httpClient = httpClient; - this.executor = Executors.newSingleThreadExecutor(); - this.listOperations = ctx.getRedisTemplate().opsForList(); - this.redisKey = constructRedisKey(); - this.trimQueue = trimQueue; - this.maxQueueSize = maxQueueSize; - init(); - } - - private void init() { - failuresCounter = new AtomicInteger(0); - future = executor.submit(() -> { - while (true) { - if (failuresCounter.get() != 0 && failuresCounter.get() % 50 == 0) { - sleep("Target HTTP server is down...", 3); - } - if (listOperations.size(redisKey) > 0) { - List list = listOperations.range(redisKey, -10, -1); - list.forEach(obj -> { - //TODO 2.5: Callback? - TbMsg msg = TbMsg.fromBytes((byte[]) obj, null); - log.debug("Trying to send the message: {}", msg); - listOperations.remove(redisKey, -1, obj); - httpClient.processMessage(ctx, msg, this); - }); - } else { - sleep("Queue is empty, waiting for tasks!", 1); - } - } - }); - } - - void destroy() { - if (future != null) { - future.cancel(true); - } - if (executor != null) { - executor.shutdownNow(); - } - } - - void push(TbMsg msg) { - listOperations.leftPush(redisKey, TbMsg.toByteArray(msg)); - if (trimQueue) { - listOperations.trim(redisKey, 0, validateMaxQueueSize()); - } - } - - void pushOnFailure(TbMsg msg) { - listOperations.rightPush(redisKey, TbMsg.toByteArray(msg)); - failuresCounter.incrementAndGet(); - } - - void resetCounter() { - failuresCounter.set(0); - } - - private String constructRedisKey() { - return ctx.getServiceId() + ctx.getSelfId(); - } - - private int validateMaxQueueSize() { - if (maxQueueSize != 0) { - return maxQueueSize; - } - return MAX_QUEUE_SIZE; - } - - private void sleep(String logMessage, int sleepSeconds) { - try { - log.debug(logMessage); - TimeUnit.SECONDS.sleep(sleepSeconds); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted!", e); - } - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java index 9b9d275977..9cb171d0dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNode.java @@ -25,8 +25,6 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import java.util.concurrent.ExecutionException; - @Slf4j @RuleNode( type = ComponentType.EXTERNAL, @@ -47,7 +45,6 @@ public class TbRestApiCallNode implements TbNode { private boolean useRedisQueueForMsgPersistence; private TbHttpClient httpClient; - private TbRedisQueueProcessor queueProcessor; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -55,20 +52,13 @@ public class TbRestApiCallNode implements TbNode { httpClient = new TbHttpClient(config); useRedisQueueForMsgPersistence = config.isUseRedisQueueForMsgPersistence(); if (useRedisQueueForMsgPersistence) { - if (ctx.getRedisTemplate() == null) { - throw new RuntimeException("Redis cache type must be used!"); - } - queueProcessor = new TbRedisQueueProcessor(ctx, httpClient, config.isTrimQueue(), config.getMaxQueueSize()); + log.warn("[{}][{}] Usage of Redis Template is deprecated starting 2.5 and will have no affect", ctx.getTenantId(), ctx.getSelfId()); } } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { - if (useRedisQueueForMsgPersistence) { - queueProcessor.push(msg); - } else { - httpClient.processMessage(ctx, msg, null); - } + public void onMsg(TbContext ctx, TbMsg msg) { + httpClient.processMessage(ctx, msg); } @Override @@ -76,9 +66,6 @@ public class TbRestApiCallNode implements TbNode { if (this.httpClient != null) { this.httpClient.destroy(); } - if (this.queueProcessor != null) { - this.queueProcessor.destroy(); - } } } From e991c0ef23b96b7bc86967d108984391ef1fa925 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 08:44:34 +0300 Subject: [PATCH 069/196] No more Synchronization nodes --- .../server/actors/ActorSystemContext.java | 6 - .../actors/ruleChain/DefaultTbContext.java | 6 - .../BaseRuleChainTransactionService.java | 225 ------------------ .../transaction/TbTransactionTask.java | 44 ---- .../thingsboard/server/common/msg/TbMsg.java | 30 +-- .../common/msg/TbMsgTransactionData.java | 30 --- common/message/src/main/proto/tbmsg.proto | 9 +- .../api/RuleChainTransactionService.java | 28 --- .../rule/engine/api/TbContext.java | 2 - .../TbSynchronizationBeginNode.java | 24 +- .../transaction/TbSynchronizationEndNode.java | 11 +- 11 files changed, 13 insertions(+), 402 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java delete mode 100644 rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index fb4ff77359..7e9b68882b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -36,7 +36,6 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; @@ -67,7 +66,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -246,10 +244,6 @@ public class ActorSystemContext { @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; - @Autowired(required = false) - @Getter - private RuleChainTransactionService ruleChainTransactionService; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 13f7b1122d..b268236448 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -24,7 +24,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.RedisTemplate; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -378,11 +377,6 @@ class DefaultTbContext implements TbContext { return mainCtx.getEntityViewService(); } - @Override - public RuleChainTransactionService getRuleChainTransactionService() { - return mainCtx.getRuleChainTransactionService(); - } - @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java b/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java deleted file mode 100644 index 477f115a96..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/BaseRuleChainTransactionService.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.RuleChainTransactionService; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.executors.DbCallbackExecutorService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - -@Service -@TbRuleEngineComponent -@Slf4j -public class BaseRuleChainTransactionService implements RuleChainTransactionService { - - private final DbCallbackExecutorService callbackExecutor; - - @Value("${actors.rule.transaction.queue_size}") - private int finalQueueSize; - @Value("${actors.rule.transaction.duration}") - private long duration; - - private final Lock transactionLock = new ReentrantLock(); - private final ConcurrentMap> transactionMap = new ConcurrentHashMap<>(); - private final Queue timeoutQueue = new ConcurrentLinkedQueue<>(); - - private ExecutorService timeoutExecutor; - - public BaseRuleChainTransactionService(DbCallbackExecutorService callbackExecutor) { - this.callbackExecutor = callbackExecutor; - } - - @PostConstruct - public void init() { - timeoutExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("rule-chain-transaction")); - executeOnTimeout(); - } - - @PreDestroy - public void destroy() { - if (timeoutExecutor != null) { - timeoutExecutor.shutdownNow(); - } - } - - @Override - public void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask transactionTask = new TbTransactionTask(msg, onStart, onEnd, onFailure, System.currentTimeMillis() + duration); - int queueSize = queue.size(); - if (queueSize >= finalQueueSize) { - log.trace("Queue has no space: {}", transactionTask); - executeOnFailure(transactionTask.getOnFailure(), "Queue has no space!"); - } else { - addMsgToQueues(queue, transactionTask); - if (queueSize == 0) { - executeOnSuccess(transactionTask.getOnStart(), transactionTask.getMsg()); - } else { - log.trace("Msg [{}][{}] is waiting to start transaction!", msg.getId(), msg.getType()); - } - } - } finally { - transactionLock.unlock(); - } - } - - @Override - public void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - //TODO 2.5 -// Optional address = routingService.resolveById(msg.getTransactionData().getOriginatorId()); -// if (address.isPresent()) { -// sendTransactionEventToRemoteServer(msg, address.get()); -// executeOnSuccess(onSuccess, msg); -// } else { - endLocalTransaction(msg, onSuccess, onFailure); -// } - } - - private void addMsgToQueues(BlockingQueue queue, TbTransactionTask transactionTask) { - queue.offer(transactionTask); - timeoutQueue.offer(transactionTask); - log.trace("Added msg to queue, size: [{}]", queue.size()); - } - - private void endLocalTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure) { - transactionLock.lock(); - try { - BlockingQueue queue = transactionMap.computeIfAbsent(msg.getTransactionData().getOriginatorId(), id -> - new LinkedBlockingQueue<>(finalQueueSize)); - - TbTransactionTask currentTransactionTask = queue.peek(); - if (currentTransactionTask != null) { - if (currentTransactionTask.getMsg().getTransactionData().getTransactionId().equals(msg.getTransactionData().getTransactionId())) { - currentTransactionTask.setCompleted(true); - queue.poll(); - log.trace("Removed msg from queue, size [{}]", queue.size()); - - executeOnSuccess(currentTransactionTask.getOnEnd(), currentTransactionTask.getMsg()); - executeOnSuccess(onSuccess, msg); - - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } else { - log.trace("Task has expired!"); - executeOnFailure(onFailure, "Task has expired!"); - } - } else { - log.trace("Queue is empty, previous task has expired!"); - executeOnFailure(onFailure, "Queue is empty, previous task has expired!"); - } - } finally { - transactionLock.unlock(); - } - } - - private void executeOnTimeout() { - timeoutExecutor.submit(() -> { - while (true) { - TbTransactionTask transactionTask = timeoutQueue.peek(); - if (transactionTask != null) { - long sleepDuration = 0L; - transactionLock.lock(); - try { - if (transactionTask.isCompleted()) { - timeoutQueue.poll(); - } else { - long expIn = transactionTask.getExpirationTime() - System.currentTimeMillis(); - if (expIn < 0) { - log.trace("Task has expired! Deleting it...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - timeoutQueue.poll(); - executeOnFailure(transactionTask.getOnFailure(), "Task has expired!"); - - BlockingQueue queue = transactionMap.get(transactionTask.getMsg().getTransactionData().getOriginatorId()); - if (queue != null) { - queue.poll(); - TbTransactionTask nextTransactionTask = queue.peek(); - if (nextTransactionTask != null) { - executeOnSuccess(nextTransactionTask.getOnStart(), nextTransactionTask.getMsg()); - } - } - } else { - sleepDuration = Math.min(expIn, duration); - } - } - } finally { - transactionLock.unlock(); - } - if (sleepDuration > 0L) { - try { - log.trace("Task has not expired! Continue executing...[{}][{}]", transactionTask.getMsg().getId(), transactionTask.getMsg().getType()); - TimeUnit.MILLISECONDS.sleep(sleepDuration); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } else { - try { - log.trace("Queue is empty, waiting for tasks!"); - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - throw new IllegalStateException("Thread interrupted", e); - } - } - } - }); - } - - private void executeOnFailure(Consumer onFailure, String exception) { - executeCallback(() -> { - onFailure.accept(new RuntimeException(exception)); - return null; - }); - } - - private void executeOnSuccess(Consumer onSuccess, TbMsg tbMsg) { - executeCallback(() -> { - onSuccess.accept(tbMsg); - return null; - }); - } - - private void executeCallback(Callable task) { - callbackExecutor.executeAsync(task); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java b/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java deleted file mode 100644 index c86f882ff7..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/transaction/TbTransactionTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.transaction; - -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.function.Consumer; - -@Data -@AllArgsConstructor -public final class TbTransactionTask { - - private final TbMsg msg; - private final Consumer onStart; - private final Consumer onEnd; - private final Consumer onFailure; - private final long expirationTime; - - private boolean isCompleted; - - public TbTransactionTask(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure, long expirationTime) { - this.msg = msg; - this.onStart = onStart; - this.onEnd = onEnd; - this.onFailure = onFailure; - this.expirationTime = expirationTime; - this.isCompleted = false; - } -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 84110ed163..2e518df084 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -44,7 +44,6 @@ public final class TbMsg implements Serializable { private final TbMsgMetaData metaData; private final TbMsgDataType dataType; private final String data; - private final TbMsgTransactionData transactionData; private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; //This field is not serialized because we use queues and there is no need to do it @@ -68,27 +67,22 @@ public final class TbMsg implements Serializable { public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), - data, origMsg.getTransactionData(), origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); + data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); } public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); + return new TbMsg(UUID.randomUUID(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); } private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { - this(id, type, originator, metaData, dataType, data, new TbMsgTransactionData(id, originator), ruleChainId, ruleNodeId, callback); - } - - private TbMsg(UUID id, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - TbMsgTransactionData transactionData, RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { this.id = id; this.type = type; this.originator = originator; this.metaData = metaData; this.dataType = dataType; this.data = data; - this.transactionData = transactionData; this.ruleChainId = ruleChainId; this.ruleNodeId = ruleNodeId; if (callback != null) { @@ -125,15 +119,6 @@ public final class TbMsg implements Serializable { builder.setMetaData(MsgProtos.TbMsgMetaDataProto.newBuilder().putAllData(msg.getMetaData().getData()).build()); } - TbMsgTransactionData transactionData = msg.getTransactionData(); - if (transactionData != null) { - MsgProtos.TbMsgTransactionDataProto.Builder transactionBuilder = MsgProtos.TbMsgTransactionDataProto.newBuilder(); - transactionBuilder.setId(transactionData.getTransactionId().toString()); - transactionBuilder.setEntityType(transactionData.getOriginatorId().getEntityType().name()); - transactionBuilder.setEntityIdMSB(transactionData.getOriginatorId().getId().getMostSignificantBits()); - transactionBuilder.setEntityIdLSB(transactionData.getOriginatorId().getId().getLeastSignificantBits()); - builder.setTransactionData(transactionBuilder.build()); - } builder.setDataType(msg.getDataType().ordinal()); builder.setData(msg.getData()); @@ -144,9 +129,6 @@ public final class TbMsg implements Serializable { try { MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data); TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap()); - EntityId transactionEntityId = EntityIdFactory.getByTypeAndUuid(proto.getTransactionData().getEntityType(), - new UUID(proto.getTransactionData().getEntityIdMSB(), proto.getTransactionData().getEntityIdLSB())); - TbMsgTransactionData transactionData = new TbMsgTransactionData(UUID.fromString(proto.getTransactionData().getId()), transactionEntityId); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); RuleChainId ruleChainId = null; RuleNodeId ruleNodeId = null; @@ -157,17 +139,17 @@ public final class TbMsg implements Serializable { ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); } TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; - return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), transactionData, ruleChainId, ruleNodeId, callback); + return new TbMsg(UUID.fromString(proto.getId()), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } } public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, this.transactionData, ruleChainId, null, callback); + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); } public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { - return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, this.transactionData, ruleChainId, ruleNodeId, callback); + return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java deleted file mode 100644 index 5d57514275..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgTransactionData.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.io.Serializable; -import java.util.UUID; - -@Data -public final class TbMsgTransactionData implements Serializable { - - private final UUID transactionId; - private final EntityId originatorId; - -} diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 03dc17b5c9..737d5f6f57 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -23,13 +23,6 @@ message TbMsgMetaDataProto { map data = 1; } -message TbMsgTransactionDataProto { - string id = 1; - string entityType = 2; - int64 entityIdMSB = 3; - int64 entityIdLSB = 4; -} - message TbMsgProto { string id = 1; string type = 2; @@ -46,7 +39,7 @@ message TbMsgProto { TbMsgMetaDataProto metaData = 11; - TbMsgTransactionDataProto transactionData = 12; + //Transaction Data (12) was removed in 2.5 int32 dataType = 13; string data = 14; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java deleted file mode 100644 index 1bb56adb1d..0000000000 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleChainTransactionService.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.rule.engine.api; - -import org.thingsboard.server.common.msg.TbMsg; - -import java.util.function.Consumer; - -public interface RuleChainTransactionService { - - void beginTransaction(TbMsg msg, Consumer onStart, Consumer onEnd, Consumer onFailure); - - void endTransaction(TbMsg msg, Consumer onSuccess, Consumer onFailure); - -} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index daab510102..9b1edc5f5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -203,8 +203,6 @@ public interface TbContext { String getServiceId(); - RuleChainTransactionService getRuleChainTransactionService(); - EventLoopGroup getSharedEventLoop(); CassandraCluster getCassandraCluster(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index b8eff0901a..31501688eb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -25,10 +25,6 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.TbMsgDataType; -import org.thingsboard.server.common.msg.TbMsgTransactionData; - -import java.util.concurrent.ExecutionException; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -43,31 +39,17 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; "Size of the queue per originator and timeout values are configurable on a system level", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbNodeEmptyConfig") +@Deprecated public class TbSynchronizationBeginNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - log.trace("Msg enters transaction - [{}][{}]", msg.getId(), msg.getType()); - - TbMsgTransactionData transactionData = new TbMsgTransactionData(msg.getId(), msg.getOriginator()); - //TODO 2.5: Callback? - TbMsg tbMsg = TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData()); - - ctx.getRuleChainTransactionService().beginTransaction(tbMsg, startMsg -> { - log.trace("Transaction starting...[{}][{}]", startMsg.getId(), startMsg.getType()); - ctx.tellNext(startMsg, SUCCESS); - }, endMsg -> log.trace("Transaction ended successfully...[{}][{}]", endMsg.getId(), endMsg.getType()), - throwable -> { - log.trace("Transaction failed! [{}][{}]", tbMsg.getId(), tbMsg.getType(), throwable); - ctx.tellFailure(tbMsg, throwable); - }); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index af3742a0fe..ac3d42ef36 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -40,25 +40,20 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") ) +@Deprecated public class TbSynchronizationEndNode implements TbNode { - private EmptyNodeConfiguration config; - @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override public void onMsg(TbContext ctx, TbMsg msg) { - ctx.getRuleChainTransactionService().endTransaction(msg, - successMsg -> ctx.tellNext(successMsg, SUCCESS), - throwable -> ctx.tellFailure(msg, throwable)); - log.trace("Msg left transaction - [{}][{}]", msg.getId(), msg.getType()); + log.warn("Synchronization Start/End nodes are deprecated since TB 2.5. Use queue with submit strategy SEQUENTIAL_WITHIN_ORIGINATOR instead."); + ctx.tellSuccess(msg); } @Override public void destroy() { - } } From 2a2a0f9caf3b94a1052d833c412e1f06b4f4af06 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Tue, 14 Apr 2020 11:42:37 +0300 Subject: [PATCH 070/196] Added new settings to web-camera input widget (#2616) --- .../system/widget_bundles/input_widgets.json | 8 ++--- .../app/widget/lib/web-camera-input-widget.js | 35 ++++++++++++++++--- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/application/src/main/data/json/system/widget_bundles/input_widgets.json b/application/src/main/data/json/system/widget_bundles/input_widgets.json index 2cc297b263..18765a8b54 100644 --- a/application/src/main/data/json/system/widget_bundles/input_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/input_widgets.json @@ -330,13 +330,13 @@ "name": "Web Camera Input", "descriptor": { "type": "latest", - "sizeX": 9.5, - "sizeY": 6.5, + "sizeX": 7.5, + "sizeY": 3, "resources": [], "templateHtml": "\n", - "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "templateCss": "", "controllerScript": "self.onInit = function() {\n var scope = self.ctx.$scope;\n scope.ctx = self.ctx;\n}\n\nself.onDataUpdated = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1\n }\n}\n\nself.onDestroy = function() {\n}\n", - "settingsSchema": "{}", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Web Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}", "dataKeySettingsSchema": "{}\n", "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" } diff --git a/ui/src/app/widget/lib/web-camera-input-widget.js b/ui/src/app/widget/lib/web-camera-input-widget.js index 510fafb3c5..e66c88b5f7 100644 --- a/ui/src/app/widget/lib/web-camera-input-widget.js +++ b/ui/src/app/widget/lib/web-camera-input-widget.js @@ -52,6 +52,11 @@ function WebCameraWidgetController($element, $scope, $window, types, utils, attr let canvas = null; let photoCamera = null; let dataKeyType = ""; + let width = 640; + let height = 480; + + const DEFAULT_IMAGE_TYPE = 'image/jpeg'; + const DEFAULT_IMAGE_QUALITY = 0.92; vm.getStream = getStream; vm.createPhoto = createPhoto; @@ -79,6 +84,8 @@ function WebCameraWidgetController($element, $scope, $window, types, utils, attr vm.isEntityDetected = true; } } + width = vm.ctx.settings.maxWidth ? vm.ctx.settings.maxWidth : 640; + height = vm.ctx.settings.maxHeight ? vm.ctx.settings.maxWidth : 480; if (datasource.dataKeys.length) { $scope.currentKey = datasource.dataKeys[0].name; dataKeyType = datasource.dataKeys[0].type; @@ -93,6 +100,24 @@ function WebCameraWidgetController($element, $scope, $window, types, utils, attr } }); + function getVideoAspectRatio() { + if (videoElement.videoWidth && videoElement.videoWidth > 0 && + videoElement.videoHeight && videoElement.videoHeight > 0) { + return videoElement.videoWidth / videoElement.videoHeight; + } + return width / height; + } + + vm.videoWidth = function() { + const videoRatio = getVideoAspectRatio(); + return Math.min(width, height * videoRatio); + } + + vm.videoHeight = function() { + const videoRatio = getVideoAspectRatio(); + return Math.min(height, width / videoRatio); + } + function hasGetUserMedia() { return !!($window.navigator.mediaDevices && $window.navigator.mediaDevices.getUserMedia); } @@ -157,10 +182,12 @@ function WebCameraWidgetController($element, $scope, $window, types, utils, attr } function createPhoto() { - canvas.width = videoElement.videoWidth; - canvas.height = videoElement.videoHeight; - canvas.getContext('2d').drawImage(videoElement, 0, 0); - vm.previewPhoto = canvas.toDataURL('image/png'); + canvas.width = vm.videoWidth(); + canvas.height = vm.videoHeight(); + canvas.getContext('2d').drawImage(videoElement, 0, 0, vm.videoWidth(), vm.videoHeight()); + const mimeType = vm.ctx.settings.imageFormat ? vm.ctx.settings.imageFormat : DEFAULT_IMAGE_TYPE; + const quality = vm.ctx.settings.imageQuality ? vm.ctx.settings.imageQuality : DEFAULT_IMAGE_QUALITY; + vm.previewPhoto = canvas.toDataURL(mimeType, quality); vm.isPreviewPhoto = true; } From 69c42d243bfc62baef2cedf5f1b3bc81c8fda97e Mon Sep 17 00:00:00 2001 From: nordmif Date: Tue, 14 Apr 2020 14:15:57 +0300 Subject: [PATCH 071/196] changed device attributes name to clarify its functionality --- .../thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java index 653b571b4b..03f0652c3f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNode.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.msg.TbMsg; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "device attributes", + name = "related device attributes", configClazz = TbGetDeviceAttrNodeConfiguration.class, nodeDescription = "Add Originators Related Device Attributes and Latest Telemetry value into Message Metadata", nodeDetails = "If Attributes enrichment configured, CLIENT/SHARED/SERVER attributes are added into Message metadata " + From 0c54f836b33a5720635b7b1216424d7a14fd7c42 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 16:17:08 +0300 Subject: [PATCH 072/196] Fixed tests --- .../server/actors/app/AppActor.java | 29 +++++++++++---- .../server/actors/tenant/TenantActor.java | 17 ++++++--- .../server/controller/BaseController.java | 9 ++++- .../state/DefaultDeviceStateService.java | 2 ++ .../DefaultRuleEngineStatisticsService.java | 21 +++++++---- application/src/main/resources/logback.xml | 1 - .../src/main/resources/thingsboard.yml | 8 ++--- .../controller/AbstractControllerTest.java | 2 ++ .../BaseEntityViewControllerTest.java | 18 ++++++++-- .../AbstractMqttTelemetryIntegrationTest.java | 17 +++++---- ...AbstractRuleEngineFlowIntegrationTest.java | 36 +++++++++---------- ...actRuleEngineLifecycleIntegrationTest.java | 23 ++++++------ application/src/test/resources/logback.xml | 2 +- .../thingsboard/server/common/msg/TbMsg.java | 4 +++ .../common/DefaultTbQueueRequestTemplate.java | 3 +- .../server/queue/kafka/TBKafkaAdmin.java | 25 +------------ .../server/queue/memory/InMemoryStorage.java | 9 ++--- .../settings/TbQueueRuleEngineSettings.java | 5 +-- common/queue/src/main/proto/queue.proto | 14 ++++---- .../test/resources/cassandra-test.properties | 1 + dao/src/test/resources/sql-test.properties | 2 ++ .../rule/engine/action/TbAlarmNodeTest.java | 12 ++++++- .../TbGetCustomerAttributeNodeTest.java | 4 +-- .../transform/TbTransformMsgNodeTest.java | 2 +- 24 files changed, 159 insertions(+), 107 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 7e4773aa8b..b7782a6a87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -42,21 +42,26 @@ import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import scala.concurrent.duration.Duration; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; public class AppActor extends ContextAwareActor { private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); private final TenantService tenantService; private final BiMap tenantActors; + private final Set deletedTenants; private boolean ruleChainsInitialized; private AppActor(ActorSystemContext systemContext) { super(systemContext); this.tenantService = systemContext.getTenantService(); this.tenantActors = HashBiMap.create(); + this.deletedTenants = new HashSet<>(); } @Override @@ -139,7 +144,11 @@ public class AppActor extends ContextAwareActor { if (SYSTEM_TENANT.equals(msg.getTenantId())) { msg.getTbMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, self()); + } else { + msg.getTbMsg().getCallback().onSuccess(); + } } } @@ -154,8 +163,10 @@ public class AppActor extends ContextAwareActor { } else { if (msg.getEntityId().getEntityType() == EntityType.TENANT && msg.getEvent() == ComponentLifecycleEvent.DELETED) { - log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); - ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId())); + log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); + TenantId tenantId = new TenantId(msg.getEntityId().getId()); + deletedTenants.add(tenantId); + ActorRef tenantActor = tenantActors.get(tenantId); if (tenantActor != null) { log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor); context().stop(tenantActor); @@ -172,16 +183,22 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + if (!deletedTenants.contains(msg.getTenantId())) { + getOrCreateTenantActor(msg.getTenantId()).tell(msg, ActorRef.noSender()); + } else { + if (msg instanceof TransportToDeviceActorMsgWrapper) { + ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); + } + } } private ActorRef getOrCreateTenantActor(TenantId tenantId) { return tenantActors.computeIfAbsent(tenantId, k -> { - log.debug("[{}] Creating tenant actor.", tenantId); + log.info("[{}] Creating tenant actor.", tenantId); ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId)) .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()); context().watch(tenantActor); - log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor); + log.info("[{}] Created tenant actor: {}.", tenantId, tenantActor); return tenantActor; }); } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index a1bf343efe..894c573e15 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -66,6 +66,8 @@ public class TenantActor extends RuleChainManagerActor { return strategy; } + boolean cantFindTenant = false; + @Override public void preStart() { log.info("[{}] Starting tenant actor.", tenantId); @@ -78,10 +80,14 @@ public class TenantActor extends RuleChainManagerActor { isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); if (isRuleEngineForCurrentTenant) { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; } } log.info("[{}] Tenant actor started.", tenantId); @@ -97,6 +103,9 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { + if (cantFindTenant) { + log.info("Missing Tenant msg: {}", msg); + } switch (msg.getMsgType()) { case PARTITION_CHANGE_MSG: PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 73b25d4de2..8385e61a20 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; @@ -667,7 +668,13 @@ public abstract class BaseController { } } TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode)); - tbClusterService.pushMsgToRuleEngine(user.getTenantId(), entityId, tbMsg, null); + TenantId tenantId = user.getTenantId(); + if (tenantId.isNullUid()) { + if (entity instanceof HasTenantId) { + tenantId = ((HasTenantId) entity).getTenantId(); + } + } + tbClusterService.pushMsgToRuleEngine(tenantId, entityId, tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 1e3ab73a26..85a4ac2669 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -325,6 +325,8 @@ public class DefaultDeviceStateService implements DeviceStateService { }); }); + addedPartitions.forEach(tpi -> partitionedDevices.computeIfAbsent(tpi, key -> ConcurrentHashMap.newKeySet())); + //TODO 3.0: replace this dummy search with new functionality to search by partitions using SQL capabilities. // Adding only devices that are in new partitions List tenants = tenantService.findTenants(new TextPageLink(Integer.MAX_VALUE)).getData(); diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 38f2fe367a..3951703351 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -76,13 +77,19 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS String queueName = ruleEngineStats.getQueueName(); ruleEngineStats.getTenantStats().forEach((id, stats) -> { TenantId tenantId = new TenantId(id); - AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); - if (stats.getTotalMsgCounter().get() > 0) { - List tsList = stats.getCounters().entrySet().stream() - .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) - .collect(Collectors.toList()); - if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + try { + AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + if (stats.getTotalMsgCounter().get() > 0) { + List tsList = stats.getCounters().entrySet().stream() + .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) + .collect(Collectors.toList()); + if (!tsList.isEmpty()) { + tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + } + } + } catch (DataValidationException e) { + if (!e.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e; } } }); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 08a976592d..614f85ba0e 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -27,7 +27,6 @@ - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b5e875e07b..c138c23fc9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -31,7 +31,7 @@ server: key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}" # Alias that identifies the key in the key store key-alias: "${SSL_KEY_ALIAS:tomcat}" - log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}" + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" ws: send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" limits: @@ -412,7 +412,7 @@ audit-log: state: defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" - persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:true}" + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" js: evaluator: "${JS_EVALUATOR:local}" # local/remote @@ -513,7 +513,7 @@ swagger: version: "${SWAGGER_VERSION:2.0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -588,7 +588,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}" queues: - - name: "Main" + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 3e9f8c7853..26f375c464 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -33,6 +33,7 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; @@ -197,6 +198,7 @@ public abstract class AbstractControllerTest { createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD); logout(); + log.info("Executed setup"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 1eb821f06a..e60f549610 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.driver.core.utils.UUIDs; import com.fasterxml.jackson.core.type.TypeReference; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.eclipse.paho.client.mqttv3.MqttAsyncClient; import org.eclipse.paho.client.mqttv3.MqttConnectOptions; @@ -24,6 +25,7 @@ import org.eclipse.paho.client.mqttv3.MqttMessage; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -46,6 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -55,6 +58,7 @@ import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; +@Slf4j public abstract class BaseEntityViewControllerTest extends AbstractControllerTest { private IdComparator idComparator; @@ -417,12 +421,22 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); - + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); +// client.disconnect(); + } + + private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { + long start = System.currentTimeMillis(); + while (!client.isConnected()) { + Thread.sleep(100); + if (start + ms < System.currentTimeMillis()) { + throw new RuntimeException("Client is not connected!"); + } + } } private Set getTelemetryKeys(String type, String id) throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f1580c65c7..122bef2e26 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -80,7 +80,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); Thread.sleep(1000); - List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); + List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4"); @@ -88,7 +88,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals(expectedKeySet, actualKeySet); - String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); + String getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); Map>> values = doGetAsync(getTelemetryValuesUrl, Map.class); assertEquals("value1", values.get("key1").get(0).get("value")); @@ -104,13 +104,17 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); - client.connect(options).waitForCompletion(3000); CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); + client.connect(options).waitForCompletion(3000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); String payload = "{\"key\":\"value\"}"; - String result = doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); +// TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. +// MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) +// MqttClient <- SUB_ACK <- Transport + Thread.sleep(1000); + doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); @@ -120,8 +124,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr private final MqttAsyncClient client; private final CountDownLatch latch; - private Integer qoS; - private String payload; + private volatile Integer qoS; + private volatile String payload; String getPayload() { return payload; @@ -138,6 +142,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr @Override public void connectionLost(Throwable throwable) { + log.error("Client connection lost", throwable); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 7fe63827eb..3343d271fe 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.rules.flow; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; @@ -35,6 +38,8 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; @@ -55,7 +60,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -142,15 +147,12 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = TbMsg.newMsg( - "CUSTOM", - device.getId(), - new TbMsgMetaData(), TbMsgDataType.JSON, "{}"); - //TODO 2.5 -// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -257,17 +259,13 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = TbMsg.newMsg( - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - TbMsgDataType.JSON, - "{}"); - //TODO 2.5 -// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 78129917ef..17f17698c9 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.rules.lifecycle; +import akka.actor.ActorRef; import com.datastax.driver.core.utils.UUIDs; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -39,6 +42,8 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; @@ -58,7 +63,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac protected User tenantAdmin; @Autowired - protected ActorService actorService; + protected ActorSystemContext actorSystem; @Autowired protected AttributesService attributesService; @@ -133,17 +138,13 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Thread.sleep(1000); + TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); + TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system - TbMsg tbMsg = TbMsg.newMsg( - "CUSTOM", - device.getId(), - new TbMsgMetaData(), - TbMsgDataType.JSON, - "{}"); - //TODO 2.5 -// actorService.onMsg(new SendToClusterMsg(device.getId(), new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg))); - - Thread.sleep(3000); + actorSystem.tell(qMsg, ActorRef.noSender()); + Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 47dacce343..64eac9e61d 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 2e518df084..3edf4e5061 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -65,6 +65,10 @@ public final class TbMsg implements Serializable { return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); } + public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { + return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); + } + public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(origMsg.getId(), type, originator, metaData.copy(), origMsg.getDataType(), data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e8ebb934d3..f02ae63441 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -95,14 +95,13 @@ public class DefaultTbQueueRequestTemplate { - log.trace("Received response to Queue Template request: {}", response); byte[] requestIdHeader = response.getHeaders().get(REQUEST_ID_HEADER); UUID requestId; if (requestIdHeader == null) { log.error("[{}] Missing requestId in header and body", response); } else { requestId = bytesToUuid(requestIdHeader); - log.trace("[{}] Response received", requestId); + log.trace("[{}] Response received: {}", requestId, response); ResponseMetaData expectedResponse = pendingRequests.remove(requestId); if (expectedResponse == null) { log.trace("[{}] Invalid or stale request", requestId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java index e1ca580c28..76c977e25a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java @@ -41,7 +41,7 @@ public class TBKafkaAdmin implements TbQueueAdmin { client = AdminClient.create(settings.toProps()); } - //TODO 2.5 + //TODO 2.5 - ybondarenko Need to pass not only settings but also properties for topic creation. Somewhere in thingsboard.yml, in KV format. @Override public void createTopicIfNotExists(String topic) { try { @@ -57,29 +57,6 @@ public class TBKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } -// -// KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); -// -// ListenableFuture topicFuture = JdkFutureAdapters.listenInPoolThread(topicDescriptionFuture); -// -// return Futures.transformAsync(topicFuture, topicDescription -> { -// KafkaFuture resultFuture = createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic); -// return JdkFutureAdapters.listenInPoolThread(resultFuture); -// }); - } - - public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { - synchronized (this) { - long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout); - while (!topicExists(topic)) { - long waitMs = timeoutExpiredMs - System.currentTimeMillis(); - if (waitMs <= 0) { - throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!"); - } else { - wait(1000); - } - } - } } public CreateTopicsResult createTopic(NewTopic topic) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index c83fa02fab..f919df871e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -60,16 +60,13 @@ public final class InMemoryStorage { if (first != null) { entities = new ArrayList<>(); entities.add(first); - } else { - entities = Collections.emptyList(); List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 100); + storage.get(topic).drainTo(otherList, 999); for (TbQueueMsg other : otherList) { entities.add((T) other); } - } - if (entities.size() > 0) { - storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).addAll(entities); + } else { + entities = Collections.emptyList(); } return entities; } catch (InterruptedException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index 95540798fc..9dfe715ece 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -36,11 +36,12 @@ public class TbQueueRuleEngineSettings { private String topic; private List queues; - //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. See how ther are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers + //TODO 2.5 ybondarenko: make sure the queue names are valid to all queue providers. + // See how they are used in TbRuleEngineQueueFactory.createToRuleEngineMsgConsumer and all producers @PostConstruct public void validate() { queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> { - log.warn("Main queue is not configured in thingsboard.yml"); + log.error("Main queue is not configured in thingsboard.yml"); return new RuntimeException("No \"Main\" queue configured!"); }); } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index dac64a3478..51bc7649be 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -400,11 +400,11 @@ message ToRuleEngineNotificationMsg { /* Messages that are handled by ThingsBoard Transport Service */ message ToTransportMsg { - int64 sessionIdMSB = 1; - int64 sessionIdLSB = 2; - SessionCloseNotificationProto sessionCloseNotification = 3; - GetAttributeResponseMsg getAttributesResponse = 4; - AttributeUpdateNotificationMsg attributeUpdateNotification = 5; - ToDeviceRpcRequestMsg toDeviceRequest = 6; - ToServerRpcResponseMsg toServerResponse = 7; + int64 sessionIdMSB = 1; + int64 sessionIdLSB = 2; + SessionCloseNotificationProto sessionCloseNotification = 3; + GetAttributeResponseMsg getAttributesResponse = 4; + AttributeUpdateNotificationMsg attributeUpdateNotification = 5; + ToDeviceRpcRequestMsg toDeviceRequest = 6; + ToServerRpcResponseMsg toServerResponse = 7; } diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index af9a4b356f..51f34a08d6 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -60,3 +60,4 @@ cassandra.query.tenant_rate_limits.enabled=false cassandra.query.tenant_rate_limits.configuration=5000:1,100000:60 cassandra.query.tenant_rate_limits.print_tenant_names=false +service.type=monolith \ No newline at end of file diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 765e0da3d6..13c0fbc818 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,6 +16,8 @@ spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 +service.type=monolith + #database.ts.type=timescale #database.ts.type=sql #database.entities.type=sql diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 2fcdc6c838..61b83d647d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -25,6 +25,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import org.mockito.stubbing.Answer; @@ -47,6 +48,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import javax.script.ScriptException; import java.io.IOException; import java.util.concurrent.Callable; +import java.util.function.Consumer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -82,6 +84,11 @@ public class TbAlarmNodeTest { @Mock private ScriptEngine detailsJs; + @Captor + private ArgumentCaptor successCaptor; + @Captor + private ArgumentCaptor> failureCaptor; + private RuleChainId ruleChainId = new RuleChainId(UUIDs.timeBased()); private RuleNodeId ruleNodeId = new RuleNodeId(UUIDs.timeBased()); @@ -119,11 +126,12 @@ public class TbAlarmNodeTest { when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); when(alarmService.findLatestByOriginatorAndType(tenantId, originator, "SomeType")).thenReturn(Futures.immediateFuture(null)); - doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(any(Alarm.class)); node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -191,6 +199,8 @@ public class TbAlarmNodeTest { node.onMsg(ctx, msg); + verify(ctx).enqueue(any(), successCaptor.capture(), failureCaptor.capture()); + successCaptor.getValue().run(); verify(ctx).tellNext(any(), eq("Created")); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 211459b84c..a0a61ae30f 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -256,7 +256,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(timeseries)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "highest"); } @@ -268,7 +268,7 @@ public class TbGetCustomerAttributeNodeTest { .thenReturn(Futures.immediateFuture(attributes)); node.onMsg(ctx, msg); - verify(ctx).tellNext(msg, SUCCESS); + verify(ctx).tellSuccess(msg); assertEquals(msg.getMetaData().getValue("tempo"), "high"); } } \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index 3d40f3acd0..a00e97cf1c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -77,7 +77,7 @@ public class TbTransformMsgNodeTest { node.onMsg(ctx, msg); verify(ctx).getDbCallbackExecutor(); ArgumentCaptor captor = ArgumentCaptor.forClass(TbMsg.class); - verify(ctx).tellNext(captor.capture(), eq(SUCCESS)); + verify(ctx).tellSuccess(captor.capture()); TbMsg actualMsg = captor.getValue(); assertEquals(transformedMsg, actualMsg); } From e1f914e2fad9917e1a60c6681ac5837a6675495c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 14 Apr 2020 16:47:01 +0300 Subject: [PATCH 073/196] Restored log level --- application/src/test/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 64eac9e61d..47dacce343 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + From e8b94d566cff1f7c85d6707370a535041f420da2 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 15 Apr 2020 16:57:40 +0300 Subject: [PATCH 074/196] init commit --- ...rvice.java => AbstractCleanUpService.java} | 40 ++++--------- .../ttl/events/EventsCleanUpService.java | 57 +++++++++++++++++++ .../AbstractTimeseriesCleanUpService.java | 49 ++++++++++++++++ .../PsqlTimeseriesCleanUpService.java | 2 +- .../TimescaleTimeseriesCleanUpService.java | 2 +- .../src/main/resources/thingsboard.yml | 11 +++- 6 files changed, 127 insertions(+), 34 deletions(-) rename application/src/main/java/org/thingsboard/server/service/ttl/{AbstractTimeseriesCleanUpService.java => AbstractCleanUpService.java} (67%) create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/PsqlTimeseriesCleanUpService.java (96%) rename application/src/main/java/org/thingsboard/server/service/ttl/{ => timeseries}/TimescaleTimeseriesCleanUpService.java (95%) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java similarity index 67% rename from application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 2399450d86..61d81ae0b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -17,47 +17,27 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.dao.util.PsqlDao; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; -@PsqlTsAnyDao -@Slf4j -public abstract class AbstractTimeseriesCleanUpService { - - @Value("${sql.ttl.ts_key_value_ttl}") - protected long systemTtl; - @Value("${sql.ttl.enabled}") - private boolean ttlTaskExecutionEnabled; +@Slf4j +@PsqlDao +public abstract class AbstractCleanUpService { @Value("${spring.datasource.url}") - private String dbUrl; + protected String dbUrl; @Value("${spring.datasource.username}") - private String dbUserName; + protected String dbUserName; @Value("${spring.datasource.password}") - private String dbPassword; - - @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}") - public void cleanUp() { - if (ttlTaskExecutionEnabled) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - doCleanUp(conn); - } catch (SQLException e) { - log.error("SQLException occurred during TTL task execution ", e); - } - } - } - - protected abstract void doCleanUp(Connection connection); + protected String dbPassword; protected long executeQuery(Connection conn, String query) { long removed = 0L; @@ -74,7 +54,7 @@ public abstract class AbstractTimeseriesCleanUpService { return removed; } - private void getWarnings(Statement statement) throws SQLException { + protected void getWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { log.debug("{}", warnings.getMessage()); @@ -86,4 +66,6 @@ public abstract class AbstractTimeseriesCleanUpService { } } -} \ No newline at end of file + protected abstract void doCleanUp(Connection connection); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java new file mode 100644 index 0000000000..bb7d112bb1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.events; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlDao +@Slf4j +@Service +public class EventsCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.events.events_key_value_ttl}") + private long ttl; + + @Value("${sql.ttl.events.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + + @Override + protected void doCleanUp(Connection connection) { + log.info("ttl: [{}]", ttl); + log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); + // TODO: 4/15/20 Do a clean up. + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java new file mode 100644 index 0000000000..75b07b9176 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/AbstractTimeseriesCleanUpService.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +@PsqlTsAnyDao +@Slf4j +public abstract class AbstractTimeseriesCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.ts.ts_key_value_ttl}") + protected long systemTtl; + + @Value("${sql.ttl.ts.enabled}") + private boolean ttlTaskExecutionEnabled; + + @Scheduled(initialDelayString = "${sql.ttl.ts.execution_interval_ms}", fixedDelayString = "${sql.ttl.ts.execution_interval_ms}") + public void cleanUp() { + if (ttlTaskExecutionEnabled) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + doCleanUp(conn); + } catch (SQLException e) { + log.error("SQLException occurred during TTL task execution ", e); + } + } + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index ab344dc518..cd403ee3b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java rename to application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java index 1dbdb4ad55..f5898b9b20 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.ttl; +package org.thingsboard.server.service.ttl.timeseries; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0402c91387..9283f9283c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -217,9 +217,14 @@ sql: # Specify Interval size for new data chunks storage. chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}" ttl: - enabled: "${SQL_TTL_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds - ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds + ts: + enabled: "${SQL_TTL_TS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds + events: + enabled: "${SQL_TTL_EVENTS_ENABLED:true}" + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_key_value_ttl: "${SQL_TTL_EVENTS_EVENTS_KEY_VALUE_TTL:0}" # Number of seconds # Actor system parameters actors: From d654e09d6a52c0a0d038ac176f36214c0dce25b2 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Wed, 15 Apr 2020 17:47:46 +0300 Subject: [PATCH 075/196] kafka topic settings * added topic-properties to kafka queue * added topic-properties to kafka queue to transport * kafka topic settings improvements --- .../src/main/resources/thingsboard.yml | 7 ++ .../server/queue/kafka/TBKafkaAdmin.java | 43 +++++++----- .../queue/kafka/TBKafkaConsumerTemplate.java | 8 ++- .../queue/kafka/TBKafkaProducerTemplate.java | 26 ++++++- .../server/queue/kafka/TbKafkaSettings.java | 8 ++- .../queue/kafka/TbKafkaTopicConfigs.java | 69 +++++++++++++++++++ .../provider/KafkaMonolithQueueFactory.java | 34 +++++++-- .../provider/KafkaTbCoreQueueFactory.java | 32 +++++++-- .../KafkaTbRuleEngineQueueFactory.java | 28 ++++++-- .../KafkaTbTransportQueueFactory.java | 42 +++++++---- .../src/main/resources/tb-coap-transport.yml | 7 ++ .../src/main/resources/tb-http-transport.yml | 7 ++ .../src/main/resources/tb-mqtt-transport.yml | 7 ++ 13 files changed, 267 insertions(+), 51 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c138c23fc9..7db2224d34 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -521,6 +521,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java index 76c977e25a..69ad6286a6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java @@ -19,15 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.queue.TbQueueAdmin; import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** * Created by ashvayka on 24.09.18. @@ -35,17 +34,34 @@ import java.util.concurrent.TimeoutException; @Slf4j public class TBKafkaAdmin implements TbQueueAdmin { - AdminClient client; + private final AdminClient client; + private final Map topicConfigs; + private final Set topics = ConcurrentHashMap.newKeySet(); - public TBKafkaAdmin(TbKafkaSettings settings) { + private final short replicationFactor; + + public TBKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { client = AdminClient.create(settings.toProps()); + this.topicConfigs = topicConfigs; + + try { + topics.addAll(client.listTopics().names().get()); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to get all topics.", e); + } + + replicationFactor = settings.getReplicationFactor(); } - //TODO 2.5 - ybondarenko Need to pass not only settings but also properties for topic creation. Somewhere in thingsboard.yml, in KV format. @Override public void createTopicIfNotExists(String topic) { + if (topics.contains(topic)) { + return; + } try { - createTopic(new NewTopic(topic, 1, (short) 1)).values().get(topic).get(); + NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); + createTopic(newTopic).values().get(topic).get(); + topics.add(topic); } catch (ExecutionException ee) { if (ee.getCause() instanceof TopicExistsException) { //do nothing @@ -57,19 +73,10 @@ public class TBKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } + } public CreateTopicsResult createTopic(NewTopic topic) { return client.createTopics(Collections.singletonList(topic)); } - - private boolean topicExists(String topic) throws InterruptedException { - KafkaFuture topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic); - try { - topicDescriptionFuture.get(); - return true; - } catch (ExecutionException e) { - return false; - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java index d882153177..069f4a6455 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java @@ -23,6 +23,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; @@ -43,7 +44,7 @@ import java.util.stream.Collectors; @Slf4j public class TBKafkaConsumerTemplate implements TbQueueConsumer { - private final TBKafkaAdmin admin; + private final TbQueueAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; private volatile boolean subscribed; @@ -57,7 +58,8 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, - int maxPollRecords) { + int maxPollRecords, + TbQueueAdmin admin) { Properties props = settings.toProps(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); if (groupId != null) { @@ -70,7 +72,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon if (maxPollRecords > 0) { props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); } - this.admin = new TBKafkaAdmin(settings); + this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; this.topic = topic; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java index 2d82ed275d..f81381bc38 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java @@ -24,12 +24,15 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** @@ -46,8 +49,12 @@ public class TBKafkaProducerTemplate implements TbQueuePro @Getter private final TbKafkaSettings settings; + private final TbQueueAdmin admin; + + private final Set topics; + @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId) { + private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId, TbQueueAdmin admin) { Properties props = settings.toProps(); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); @@ -57,6 +64,8 @@ public class TBKafkaProducerTemplate implements TbQueuePro this.settings = settings; this.producer = new KafkaProducer<>(props); this.defaultTopic = defaultTopic; + this.admin = admin; + topics = ConcurrentHashMap.newKeySet(); } @Override @@ -65,6 +74,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + createTopicIfNotExist(tpi); String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; @@ -85,8 +95,18 @@ public class TBKafkaProducerTemplate implements TbQueuePro }); } + private void createTopicIfNotExist(TopicPartitionInfo tpi) { + if (topics.contains(tpi)) { + return; + } + admin.createTopicIfNotExists(tpi.getFullTopicName()); + topics.add(tpi); + } + @Override public void stop() { - + if (producer != null) { + producer.close(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index e8581d9719..66121cb215 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.kafka; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; @@ -32,9 +33,6 @@ import java.util.Properties; @Component public class TbKafkaSettings { - static final String REQUEST_ID_HEADER = "requestId"; - static final String RESPONSE_TOPIC_HEADER = "responseTopic"; - @Value("${queue.kafka.bootstrap.servers}") private String servers; @@ -53,6 +51,10 @@ public class TbKafkaSettings { @Value("${queue.kafka.buffer.memory}") private long bufferMemory; + @Value("${queue.kafka.replication_factor}") + @Getter + private short replicationFactor; + @Value("${kafka.other:#{null}}") private List other; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java new file mode 100644 index 0000000000..8682ca385f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +public class TbKafkaTopicConfigs { + @Value("${queue.kafka.topic-properties.core}") + private String coreProperties; + @Value("${queue.kafka.topic-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.kafka.topic-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.kafka.topic-properties.notifications}") + private String notificationsProperties; + @Value("${queue.kafka.topic-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreConfigs; + @Getter + private Map ruleEngineConfigs; + @Getter + private Map transportApiConfigs; + @Getter + private Map notificationsConfigs; + @Getter + private Map jsExecutorConfigs; + + @PostConstruct + private void init() { + coreConfigs = getConfigs(coreProperties); + ruleEngineConfigs = getConfigs(ruleEngineProperties); + transportApiConfigs = getConfigs(transportApiProperties); + notificationsConfigs = getConfigs(notificationsProperties); + jsExecutorConfigs = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dae4221e91..9eb656c2ea 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -40,6 +41,7 @@ import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -62,13 +64,20 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -77,6 +86,12 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -85,6 +100,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); + requestBuilder.admin(notificationAdmin); return requestBuilder.build(); } @@ -94,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -103,6 +120,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -112,6 +130,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -121,6 +140,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -133,6 +153,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); return consumerBuilder.build(); } @@ -144,6 +165,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -155,6 +177,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); return consumerBuilder.build(); } @@ -166,6 +189,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -177,6 +201,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); return consumerBuilder.build(); } @@ -186,6 +211,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); + requestBuilder.admin(transportApiAdmin); return requestBuilder.build(); } @@ -196,24 +222,24 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); responseBuilder.decoder(msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.queueAdmin(jsExecutorAdmin); builder.requestTemplate(requestBuilder.build()); builder.responseTemplate(responseBuilder.build()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index e148416bb4..8800c1c55a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -40,6 +41,7 @@ import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -59,12 +61,19 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -72,6 +81,12 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -80,6 +95,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -89,6 +105,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -98,6 +115,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -107,6 +125,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -116,6 +135,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -127,6 +147,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(coreAdmin); return consumerBuilder.build(); } @@ -138,6 +159,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -149,6 +171,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(transportApiAdmin); return consumerBuilder.build(); } @@ -158,6 +181,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -168,24 +192,24 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); responseBuilder.decoder(msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.queueAdmin(jsExecutorAdmin); builder.requestTemplate(requestBuilder.build()); builder.responseTemplate(responseBuilder.build()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 8262d27354..b96d4ebe86 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -38,6 +39,7 @@ import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -56,17 +58,28 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaTbRuleEngineQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.jsInvokeSettings = jsInvokeSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -75,6 +88,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -84,6 +98,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -93,6 +108,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -103,6 +119,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -112,6 +129,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -124,6 +142,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(ruleEngineAdmin); return consumerBuilder.build(); } @@ -135,6 +154,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); return consumerBuilder.build(); } @@ -145,24 +165,24 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); + requestBuilder.admin(jsExecutorAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("rule-engine-node-" + serviceInfoProvider.getServiceId()); -// responseBuilder.autoCommit(true); -// responseBuilder.autoCommitIntervalMs(autoCommitInterval); responseBuilder.decoder(msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + builder.queueAdmin(jsExecutorAdmin); builder.requestTemplate(requestBuilder.build()); builder.responseTemplate(responseBuilder.build()); builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 12f677af9d..f5a31cf1c7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -18,25 +18,27 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TBKafkaAdmin; import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -50,18 +52,29 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbKafkaTopicConfigs kafkaTopicConfigs) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + + this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override @@ -70,6 +83,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); + requestBuilder.admin(transportApiAdmin); TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); @@ -77,10 +91,11 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(transportApiAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(new TBKafkaAdmin(kafkaSettings)); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(requestBuilder.build()); templateBuilder.responseTemplate(responseBuilder.build()); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -93,8 +108,9 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { public TbQueueProducer> createRuleEngineMsgProducer() { TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("transport-node-rule-engine-"+ serviceInfoProvider.getServiceId()); + requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); + requestBuilder.admin(ruleEngineAdmin); return requestBuilder.build(); } @@ -104,6 +120,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(coreAdmin); return requestBuilder.build(); } @@ -115,6 +132,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + responseBuilder.admin(notificationAdmin); return responseBuilder.build(); } } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index b94148f608..e2fb7e149a 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -50,6 +50,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0fa2788cac..dfd2c14be2 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -51,6 +51,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 747427caf8..81f0fb795d 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -81,6 +81,13 @@ queue: batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" From 0a136588eb913f1b699132cb454d1ec6ea76c4f8 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 15 Apr 2020 17:48:47 +0300 Subject: [PATCH 076/196] CheckPoint Node ui, Queue Controller, queue-type-list directive (#2612) * init commit * ui: service for queue * merge with develop/2.5 & fix license * queue directive * queue directive * queue-type-list directive & TbCheckPointNode * added rulenode-core-config.js file * added new rulenode-core-config.js * Develop/2.5 check point node (#3) * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * Develop/2.5 check point node (#4) * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 * fixed mistakes to pull https://github.com/thingsboard/thingsboard/pull/2612 Co-authored-by: Dmitriymush Co-authored-by: Dmitriy Mushat <54553744+Dmitriymush@users.noreply.github.com> --- .../server/controller/QueueController.java | 54 +++++++++ .../rule/engine/flow/TbAckNode.java | 10 +- .../rule/engine/flow/TbCheckpointNode.java | 6 +- .../static/rulenode/rulenode-core-config.js | 8 +- ui/src/app/api/queue.service.js | 40 +++++++ ui/src/app/components/queue/index.js | 23 ++++ .../queue/queue-type-list.directive.js | 111 ++++++++++++++++++ .../app/components/queue/queue-type-list.scss | 15 +++ .../components/queue/queue-type-list.tpl.html | 43 +++++++ ui/src/app/layout/index.js | 2 + ui/src/app/locale/locale.constant-en_US.json | 6 + ui/src/app/locale/locale.constant-ru_RU.json | 6 + ui/src/app/locale/locale.constant-uk_UA.json | 6 + 13 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/controller/QueueController.java create mode 100644 ui/src/app/api/queue.service.js create mode 100644 ui/src/app/components/queue/index.js create mode 100644 ui/src/app/components/queue/queue-type-list.directive.js create mode 100644 ui/src/app/components/queue/queue-type-list.scss create mode 100644 ui/src/app/components/queue/queue-type-list.tpl.html diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java new file mode 100644 index 0000000000..8d1cec7f4b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +public class QueueController extends BaseController { + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET) + @ResponseBody + public List getTenantQueuesByServiceType(@RequestParam String serviceType) throws ThingsboardException { + checkParameter("serviceType", serviceType); + try { + ServiceType type = ServiceType.valueOf(serviceType); + switch (type) { + case TB_RULE_ENGINE: + return Arrays.asList("HighPriority", "Main"); + default: + return Collections.emptyList(); + } + } catch (Exception e) { + throw handleException(e); + } + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java index 9351e970ed..b5af3f1563 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbAckNode.java @@ -22,7 +22,6 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.TbRelationTypes; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -33,12 +32,17 @@ import org.thingsboard.server.common.msg.TbMsg; name = "acknowledge", configClazz = EmptyNodeConfiguration.class, nodeDescription = "Acknowledges the incoming message", - nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.") - + nodeDetails = "After acknowledgement, the message is pushed to related rule nodes. Useful if you don't care what happens to this message next.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbNodeEmptyConfig" +) public class TbAckNode implements TbNode { + EmptyNodeConfiguration config; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java index 0364251076..d5048eadbd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java @@ -35,8 +35,10 @@ import static org.thingsboard.common.util.DonAsynchron.withCallback; name = "checkpoint", configClazz = TbCheckpointNodeConfiguration.class, nodeDescription = "transfers the message to another queue", - nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.") - + nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", + uiResources = {"static/rulenode/rulenode-core-config.js"}, + configDirective = "tbActionNodeCheckPointConfig" +) public class TbCheckpointNode implements TbNode { private TbCheckpointNodeConfiguration config; diff --git a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js index fd5d9b7c62..f848a91875 100644 --- a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js +++ b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js @@ -1,6 +1,6 @@ -!function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(103)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; +!function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(105)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
tb.rulenode.select-queue-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; },function(e,t){e.exports="
tb.rulenode.interval-seconds-required
tb.rulenode.min-interval-seconds-message
tb.rulenode.output-timeseries-key-prefix-required
"},function(e,t){e.exports='
{{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
tb.rulenode.period-seconds-required
tb.rulenode.min-period-0-seconds-message
tb.rulenode.period-in-seconds-pattern-required
tb.rulenode.period-in-seconds-pattern-hint
tb.rulenode.max-pending-messages-required
tb.rulenode.max-pending-messages-range
tb.rulenode.max-pending-messages-range
'},function(e,t){e.exports="
tb.rulenode.gcp-project-id-required
tb.rulenode.pubsub-topic-name-required
{{ 'action.remove' | translate }} close
tb.rulenode.no-file
{{configuration.serviceAccountKeyFileName}}
tb.rulenode.message-attributes-hint
"},function(e,t){e.exports='
{{ property }}
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
{{ \'tb.rulenode.automatic-recovery\' | translate }}
tb.rulenode.min-connection-timeout-ms-message
tb.rulenode.min-handshake-timeout-ms-message
'},function(e,t){e.exports='
tb.rulenode.endpoint-url-pattern-required
tb.rulenode.endpoint-url-pattern-hint
 {{ type }} {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}
tb.rulenode.read-timeout-hint
tb.rulenode.max-parallel-requests-count-hint
tb.rulenode.headers-hint
{{ \'tb.rulenode.use-redis-queue\' | translate }}
{{ \'tb.rulenode.trim-redis-queue\' | translate }}
'},function(e,t){e.exports="
"},function(e,t){e.exports="
tb.rulenode.timeout-required
tb.rulenode.min-timeout-message
"},function(e,t){e.exports='
tb.rulenode.custom-table-name-required
tb.rulenode.custom-table-hint
'},function(e,t){e.exports='
{{ \'tb.rulenode.use-system-smtp-settings\' | translate }}
{{smtpProtocol.toUpperCase()}}
tb.rulenode.smtp-host-required
tb.rulenode.smtp-port-required
tb.rulenode.smtp-port-range
tb.rulenode.smtp-port-range
tb.rulenode.timeout-required
tb.rulenode.min-timeout-msec-message
 {{ \'tb.rulenode.enable-tls\' | translate }} {{tlsVersion}}
'},function(e,t){e.exports="
tb.rulenode.topic-arn-pattern-required
tb.rulenode.topic-arn-pattern-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
"},function(e,t){e.exports='
{{ type.name | translate }}
tb.rulenode.queue-url-pattern-required
tb.rulenode.queue-url-pattern-hint
tb.rulenode.min-delay-seconds-message
tb.rulenode.max-delay-seconds-message
tb.rulenode.message-attributes-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
'},function(e,t){e.exports="
tb.rulenode.default-ttl-required
tb.rulenode.min-default-ttl-message
"},function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-type
device.device-types
"},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
{{\'tb.rulenode.entity-details-\'+item.toLowerCase() | translate}} tb.rulenode.no-entity-details-matching {{\'tb.rulenode.entity-details-\'+$chip.toLowerCase() | translate}} {{ \'tb.rulenode.add-to-metadata\' | translate }}
tb.rulenode.add-to-metadata-hint
'},function(e,t){e.exports='
{{ type }}
tb.rulenode.fetch-mode-hint
{{ type }}
tb.rulenode.order-by-hint
tb.rulenode.limit-hint
{{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}
tb.rulenode.use-metadata-interval-patterns-hint
tb.rulenode.start-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.end-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.start-interval-pattern-required
tb.rulenode.start-interval-pattern-hint
tb.rulenode.end-interval-pattern-required
tb.rulenode.end-interval-pattern-hint
'; -},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},31,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(75),r=a(i),o=n(53),l=a(o),s=n(58),d=a(s),u=n(55),c=a(u),m=n(54),g=a(m),p=n(62),f=a(p),b=n(69),v=a(b),y=n(70),h=a(y),q=n(68),k=a(q),x=n(61),$=a(x),T=n(73),C=a(T),w=n(74),M=a(w),S=n(67),N=a(S),_=n(63),F=a(_),E=n(72),P=a(E),A=n(65),V=a(A),I=n(64),O=a(I),j=n(52),D=a(j),L=n(76),R=a(L),K=n(57),U=a(K),z=n(56),H=a(z),B=n(71),G=a(B),Y=n(59),Q=a(Y),W=n(66),J=a(W);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",N.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",O.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader; -t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(84),r=a(i),o=n(85),l=a(o),s=n(80),d=a(s),u=n(86),c=a(u),m=n(79),g=a(m),p=n(87),f=a(p),b=n(82),v=a(b),y=n(81),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(95),r=a(i),o=n(93),l=a(o),s=n(96),d=a(s),u=n(90),c=a(u),m=n(94),g=a(m),p=n(89),f=a(p),b=n(91),v=a(b),y=n(88),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(47),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(99),r=a(i),o=n(101),l=a(o),s=n(102),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(106),r=a(i),o=n(92),l=a(o),s=n(83),d=a(s),u=n(100),c=a(u),m=n(60),g=a(m),p=n(78),f=a(p),b=n(98),v=a(b),y=n(77),h=a(y),q=n(97),k=a(q),x=n(105),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required","endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata", -header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16"},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(104),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); +},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},32,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.serviceType="TB_RULE_ENGINE",n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(77),r=a(i),o=n(54),l=a(o),s=n(60),d=a(s),u=n(57),c=a(u),m=n(56),g=a(m),p=n(64),f=a(p),b=n(71),v=a(b),y=n(72),h=a(y),q=n(70),k=a(q),x=n(63),$=a(x),T=n(75),C=a(T),w=n(76),M=a(w),N=n(69),S=a(N),_=n(65),F=a(_),E=n(74),P=a(E),A=n(67),V=a(A),I=n(66),j=a(I),O=n(53),D=a(O),L=n(78),R=a(L),K=n(59),U=a(K),z=n(58),H=a(z),B=n(73),G=a(B),Y=n(61),Q=a(Y),W=n(68),J=a(W),Z=n(55),X=a(Z);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",S.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",j.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).directive("tbActionNodeCheckPointConfig",X.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){ +var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader;t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(86),r=a(i),o=n(87),l=a(o),s=n(82),d=a(s),u=n(88),c=a(u),m=n(81),g=a(m),p=n(89),f=a(p),b=n(84),v=a(b),y=n(83),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(43),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(97),r=a(i),o=n(95),l=a(o),s=n(98),d=a(s),u=n(92),c=a(u),m=n(96),g=a(m),p=n(91),f=a(p),b=n(93),v=a(b),y=n(90),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(101),r=a(i),o=n(103),l=a(o),s=n(104),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(52),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(108),r=a(i),o=n(94),l=a(o),s=n(85),d=a(s),u=n(102),c=a(u),m=n(62),g=a(m),p=n(80),f=a(p),b=n(100),v=a(b),y=n(79),h=a(y),q=n(99),k=a(q),x=n(107),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required", +"endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16","select-queue-hint":"The queue name can be selected from a drop-down list or add a custom name."},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(106),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); //# sourceMappingURL=rulenode-core-config.js.map \ No newline at end of file diff --git a/ui/src/app/api/queue.service.js b/ui/src/app/api/queue.service.js new file mode 100644 index 0000000000..0fe93e5fa6 --- /dev/null +++ b/ui/src/app/api/queue.service.js @@ -0,0 +1,40 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default angular.module('thingsboard.api.queue', []) + .factory('queueService', queueService) + .name; + +/*@ngInject*/ +function queueService($http, $q) { + var service = { + getTenantQueuesByServiceType: getTenantQueuesByServiceType + }; + + return service; + + function getTenantQueuesByServiceType(serviceType, config) { + let deferred = $q.defer(); + let url = '/api/tenant/queues?serviceType=' + serviceType; + + $http.get(url, config).then(function success(data) { + deferred.resolve(data); + }, function fail() { + deferred.reject(); + }); + return deferred.promise; + } +} \ No newline at end of file diff --git a/ui/src/app/components/queue/index.js b/ui/src/app/components/queue/index.js new file mode 100644 index 0000000000..5236603270 --- /dev/null +++ b/ui/src/app/components/queue/index.js @@ -0,0 +1,23 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import thingsboardApiQueue from '../../api/queue.service'; +import queueTypeList from "./queue-type-list.directive"; + +export default angular.module('thingsboard.queue', [ + thingsboardApiQueue +]) + .directive('tbQueueTypeList', queueTypeList) + .name; diff --git a/ui/src/app/components/queue/queue-type-list.directive.js b/ui/src/app/components/queue/queue-type-list.directive.js new file mode 100644 index 0000000000..6a2f99d280 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.directive.js @@ -0,0 +1,111 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import './queue-type-list.scss'; + +/* eslint-disable import/no-unresolved, import/default */ + +import queueTypeListTemplate from './queue-type-list.tpl.html'; + +/* eslint-enable import/no-unresolved, import/default */ + +/*@ngInject*/ +export default function QueueTypeList($compile, $templateCache, $q, $filter, queueService) { + + var linker = function (scope, element, attrs, ngModelCtrl) { + var template = $templateCache.get(queueTypeListTemplate); + element.html(template); + + scope.queues = null; + scope.queue = null; + scope.tbRequired = angular.isDefined(scope.tbRequired) ? scope.tbRequired : false; + scope.queueSearchText = ''; + + scope.fetchQueues = function(searchText) { + var deferred = $q.defer(); + loadQueues().then( + function success(queueArr) { + let result = $filter('filter')(queueArr, {'$': searchText}); + if (result && result.length) { + if (searchText && searchText.length && result.indexOf(searchText) === -1) { + result.push(searchText); + } + result.sort(); + deferred.resolve(result); + } else { + deferred.resolve([searchText]); + } + }, + function fail() { + deferred.reject(); + } + ); + + return deferred.promise; + }; + + scope.updateView = function () { + if (!scope.disabled) { + ngModelCtrl.$setViewValue(scope.queue); + } + }; + + function loadQueues() { + var deferred = $q.defer(); + if (!scope.queues) { + queueService.getTenantQueuesByServiceType(scope.queueType).then( + function success(queueArr) { + scope.queues = queueArr.data; + deferred.resolve(scope.queues); + }, + function fail() { + deferred.reject(); + } + ); + } else { + deferred.resolve(scope.queues); + } + return deferred.promise; + } + + ngModelCtrl.$render = function () { + scope.queue = ngModelCtrl.$viewValue; + }; + + scope.$watch('queue', function (newValue, prevValue) { + if (!angular.equals(newValue, prevValue)) { + scope.updateView(); + } + }); + + scope.$watch('disabled', function () { + scope.updateView(); + }); + + $compile(element.contents())(scope); + }; + + return { + restrict: "E", + require: "^ngModel", + link: linker, + scope: { + theForm: '=?', + tbRequired: '=?', + disabled:'=ngDisabled', + queueType: '=?' + } + }; +} \ No newline at end of file diff --git a/ui/src/app/components/queue/queue-type-list.scss b/ui/src/app/components/queue/queue-type-list.scss new file mode 100644 index 0000000000..b8f6f58fb2 --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.scss @@ -0,0 +1,15 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/ui/src/app/components/queue/queue-type-list.tpl.html b/ui/src/app/components/queue/queue-type-list.tpl.html new file mode 100644 index 0000000000..5fc700026e --- /dev/null +++ b/ui/src/app/components/queue/queue-type-list.tpl.html @@ -0,0 +1,43 @@ + + + +
+ {{item}} +
+ +
+
{{'queue.name_required' | translate}}
+
+ \ No newline at end of file diff --git a/ui/src/app/layout/index.js b/ui/src/app/layout/index.js index a00baa5199..f3b4d10e4f 100644 --- a/ui/src/app/layout/index.js +++ b/ui/src/app/layout/index.js @@ -55,6 +55,7 @@ import thingsboardEntityView from '../entity-view'; import thingsboardWidgetLibrary from '../widget'; import thingsboardDashboard from '../dashboard'; import thingsboardRuleChain from '../rulechain'; +import thingsboardQueue from '../components/queue'; import thingsboardJsonForm from '../jsonform'; @@ -86,6 +87,7 @@ export default angular.module('thingsboard.home', [ thingsboardWidgetLibrary, thingsboardDashboard, thingsboardRuleChain, + thingsboardQueue, thingsboardJsonForm, thingsboardApiDevice, thingsboardApiLogin, diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index e68ccec157..606fabd375 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1484,6 +1484,12 @@ "help": "Help", "reset-debug-mode": "Reset debug mode in all nodes" }, + "queue": { + "select_name": "Select queue name", + "name": "Queue Name", + "name_required": "Queue name is required!" + + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index f86eb988ff..e93c92124c 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1403,6 +1403,12 @@ "help": "Помощь", "reset-debug-mode": "Сбросить режим отладки во всех правилах" }, + "queue": { + "select_name": "Выберите имя для Queue", + "name": "Имя для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "tenant": { "tenant": "Владелец", "tenants": "Владельцы", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index 5a0899014d..c19049bc11 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1820,6 +1820,12 @@ "help": "Допомога", "reset-debug-mode": "Вимкнути режим налогодження у всіх правилах" }, + "queue": { + "select_name": "Виберіть ім'я для Queue", + "name": "Iм'я для Queue", + "name_required": "Поле 'Имя для Queue' обязательно к заполнению!" + + }, "scheduler": { "scheduler": "Планувальник", "scheduler-event": "Подія планувальника", From 9fa82078a02d56ce1f3835bc6788b97ed52c54c8 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 12:43:10 +0300 Subject: [PATCH 077/196] created TbAwsSqsQueueAttributes and added queue properties to yml --- .../src/main/resources/thingsboard.yml | 8 +- .../server/queue/TbQueueAdmin.java | 1 + .../azure/servicebus/TbServiceBusAdmin.java | 4 +- .../{TBKafkaAdmin.java => TbKafkaAdmin.java} | 11 ++- ...late.java => TbKafkaConsumerTemplate.java} | 4 +- .../server/queue/kafka/TbKafkaHandler.java | 27 ------ .../queue/kafka/TbKafkaPartitioner.java | 30 ------- ...late.java => TbKafkaProducerTemplate.java} | 4 +- .../queue/kafka/TbKafkaTopicConfigs.java | 2 + .../provider/AwsSqsMonolithQueueFactory.java | 40 +++++---- .../provider/AwsSqsTbCoreQueueFactory.java | 36 +++++--- .../AwsSqsTbRuleEngineQueueFactory.java | 30 ++++--- .../provider/AwsSqsTransportQueueFactory.java | 24 ++++-- .../InMemoryTbTransportQueueFactory.java | 11 ++- .../provider/KafkaMonolithQueueFactory.java | 42 +++++----- .../provider/KafkaTbCoreQueueFactory.java | 38 ++++----- .../KafkaTbRuleEngineQueueFactory.java | 32 +++---- .../KafkaTbTransportQueueFactory.java | 24 +++--- .../server/queue/pubsub/TbPubSubAdmin.java | 5 ++ .../queue/rabbitmq/TbRabbitMqAdmin.java | 7 +- .../server/queue/sqs/TbAwsSqsAdmin.java | 59 +++++++------ .../queue/sqs/TbAwsSqsQueueAttributes.java | 83 +++++++++++++++++++ .../server/queue/sqs/TbAwsSqsSettings.java | 2 - .../src/main/resources/tb-coap-transport.yml | 8 +- .../src/main/resources/tb-http-transport.yml | 8 +- .../src/main/resources/tb-mqtt-transport.yml | 8 +- 26 files changed, 334 insertions(+), 214 deletions(-) rename common/queue/src/main/java/org/thingsboard/server/queue/kafka/{TBKafkaAdmin.java => TbKafkaAdmin.java} (92%) rename common/queue/src/main/java/org/thingsboard/server/queue/kafka/{TBKafkaConsumerTemplate.java => TbKafkaConsumerTemplate.java} (97%) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java rename common/queue/src/main/java/org/thingsboard/server/queue/kafka/{TBKafkaProducerTemplate.java => TbKafkaProducerTemplate.java} (94%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7db2224d34..e1296275cd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -533,7 +533,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index deb84f440e..bb541fb626 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -19,4 +19,5 @@ public interface TbQueueAdmin { void createTopicIfNotExists(String topic); + void destroy(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index fb9cd89a8d..336514b7b2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -23,7 +23,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -68,8 +67,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } } - @PreDestroy - private void destroy() { + public void destroy() { try { client.close(); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java similarity index 92% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 69ad6286a6..b92b094af1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -32,7 +32,7 @@ import java.util.concurrent.ExecutionException; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaAdmin implements TbQueueAdmin { +public class TbKafkaAdmin implements TbQueueAdmin { private final AdminClient client; private final Map topicConfigs; @@ -40,7 +40,7 @@ public class TBKafkaAdmin implements TbQueueAdmin { private final short replicationFactor; - public TBKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { + public TbKafkaAdmin(TbKafkaSettings settings, Map topicConfigs) { client = AdminClient.create(settings.toProps()); this.topicConfigs = topicConfigs; @@ -76,6 +76,13 @@ public class TBKafkaAdmin implements TbQueueAdmin { } + @Override + public void destroy() { + if (client != null) { + client.close(); + } + } + public CreateTopicsResult createTopic(NewTopic topic) { return client.createTopics(Collections.singletonList(topic)); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java similarity index 97% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 069f4a6455..49d1d74102 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -42,7 +42,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaConsumerTemplate implements TbQueueConsumer { +public class TbKafkaConsumerTemplate implements TbQueueConsumer { private final TbQueueAdmin admin; private final KafkaConsumer consumer; @@ -55,7 +55,7 @@ public class TBKafkaConsumerTemplate implements TbQueueCon private final String topic; @Builder - private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, + private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords, diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java deleted file mode 100644 index 8c2cbd2771..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaHandler.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import com.google.common.util.concurrent.ListenableFuture; - -/** - * Created by ashvayka on 05.10.18. - */ -public interface TbKafkaHandler { - - ListenableFuture handle(Request request); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java deleted file mode 100644 index b31f87642b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaPartitioner.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.kafka; - -import org.apache.kafka.clients.producer.Partitioner; -import org.apache.kafka.common.PartitionInfo; - -import java.util.List; - -/** - * Created by ashvayka on 25.09.18. - */ -public interface TbKafkaPartitioner extends Partitioner { - - int partition(String topic, String key, T value, byte[] encodedValue, List partitions); - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java similarity index 94% rename from common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java rename to common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index f81381bc38..4f26f51da7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TBKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -39,7 +39,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TBKafkaProducerTemplate implements TbQueueProducer { +public class TbKafkaProducerTemplate implements TbQueueProducer { private final KafkaProducer producer; @@ -54,7 +54,7 @@ public class TBKafkaProducerTemplate implements TbQueuePro private final Set topics; @Builder - private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaPartitioner partitioner, String defaultTopic, String clientId, TbQueueAdmin admin) { + private TbKafkaProducerTemplate(TbKafkaSettings settings, String defaultTopic, String clientId, TbQueueAdmin admin) { Properties props = settings.toProps(); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 8682ca385f..4b68abf929 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.kafka; import lombok.Getter; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -24,6 +25,7 @@ import java.util.HashMap; import java.util.Map; @Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") public class TbKafkaTopicConfigs { @Value("${queue.kafka.topic-properties.core}") private String coreProperties; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 26da11aa3b..90fe643505 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -36,6 +36,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -49,14 +50,20 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public AwsSqsMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings) { + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,69 +71,74 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; - admin = new TbAwsSqsAdmin(sqsSettings); + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getResponsesTopic()); + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 87aba51843..e169e6502d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -40,6 +40,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -52,70 +53,81 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, TbQueueCoreSettings coreSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = new TbAwsSqsAdmin(sqsSettings); + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, coreSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic(), + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index d83b760c19..33f1d8c113 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -48,54 +49,63 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public AwsSqsTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings) { + TbAwsSqsSettings sqsSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.sqsSettings = sqsSettings; - admin = new TbAwsSqsAdmin(sqsSettings); + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); + this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, coreSettings.getTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, ruleEngineSettings.getTopic(), + return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 4196b226ee..644bc6b709 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -35,6 +35,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Component @@ -44,33 +45,38 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; - admin = new TbAwsSqsAdmin(sqsSettings); this.serviceInfoProvider = serviceInfoProvider; + + this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbAwsSqsProducerTemplate> producerTemplate = - new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); TbAwsSqsConsumerTemplate> consumerTemplate = - new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, + new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,17 +87,17 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(admin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbAwsSqsConsumerTemplate<>(admin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), + return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 29660f2461..c03f5e52d8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -33,6 +34,7 @@ import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -60,8 +62,15 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(topic -> { + + templateBuilder.queueAdmin(new TbQueueAdmin() { + @Override + public void createTopicIfNotExists(String topic) {} + + @Override + public void destroy() {} }); + templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 9eb656c2ea..72c6f7b8b9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -37,9 +37,9 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -87,16 +87,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportNotificationSettings.getNotificationsTopic()); @@ -106,7 +106,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -116,7 +116,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -126,7 +126,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -136,7 +136,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -147,7 +147,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { String queueName = configuration.getName(); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); @@ -159,7 +159,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -171,7 +171,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); @@ -183,7 +183,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -195,7 +195,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); consumerBuilder.clientId("monolith-transport-api-consumer-" + serviceInfoProvider.getServiceId()); @@ -207,7 +207,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueProducer> createTransportApiResponseProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("monolith-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getResponsesTopic()); @@ -218,13 +218,13 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); requestBuilder.admin(jsExecutorAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 8800c1c55a..ca10830981 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -37,9 +37,9 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -82,16 +82,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -101,7 +101,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -111,7 +111,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -121,7 +121,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -131,7 +131,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -141,7 +141,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToCoreMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); @@ -153,7 +153,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("tb-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -165,7 +165,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(transportApiSettings.getRequestsTopic()); consumerBuilder.clientId("tb-core-transport-api-consumer-" + serviceInfoProvider.getServiceId()); @@ -177,7 +177,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTransportApiResponseProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-core-transport-api-producer-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -188,13 +188,13 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); requestBuilder.admin(jsExecutorAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b96d4ebe86..bcac399a64 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -35,9 +35,9 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -76,15 +76,15 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.jsInvokeSettings = jsInvokeSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-transport-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -94,7 +94,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -104,7 +104,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-rule-engine-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -115,7 +115,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -125,7 +125,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("tb-rule-engine-to-core-notifications-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -136,7 +136,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { String queueName = configuration.getName(); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(ruleEngineSettings.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); @@ -148,7 +148,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> consumerBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName()); consumerBuilder.clientId("tb-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); @@ -161,13 +161,13 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("producer-js-invoke-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(jsInvokeSettings.getRequestTopic()); requestBuilder.admin(jsExecutorAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("js-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index f5a31cf1c7..2f7aab4779 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -30,9 +30,9 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TBKafkaAdmin; -import org.thingsboard.server.queue.kafka.TBKafkaConsumerTemplate; -import org.thingsboard.server.queue.kafka.TBKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -71,21 +71,21 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.coreAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); - this.transportApiAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TBKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); + this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-api-request-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(transportApiSettings.getRequestsTopic()); requestBuilder.admin(transportApiAdmin); - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("transport-api-response-" + serviceInfoProvider.getServiceId()); @@ -106,7 +106,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-rule-engine-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(ruleEngineSettings.getTopic()); @@ -116,7 +116,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder> requestBuilder = TBKafkaProducerTemplate.builder(); + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); requestBuilder.settings(kafkaSettings); requestBuilder.clientId("transport-node-core-" + serviceInfoProvider.getServiceId()); requestBuilder.defaultTopic(coreSettings.getTopic()); @@ -126,7 +126,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder> responseBuilder = TBKafkaConsumerTemplate.builder(); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> responseBuilder = TbKafkaConsumerTemplate.builder(); responseBuilder.settings(kafkaSettings); responseBuilder.topic(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); responseBuilder.clientId("transport-api-notifications-" + serviceInfoProvider.getServiceId()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index f0af639d4d..cd575818f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -121,6 +121,11 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } + @Override + public void destroy() { + + } + private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index fbd678045b..9a7e5db342 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -14,15 +14,14 @@ * limitations under the License. */ package org.thingsboard.server.queue.rabbitmq; + import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; - import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; -import javax.annotation.PreDestroy; import java.io.IOException; import java.util.concurrent.TimeoutException; @@ -62,8 +61,8 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { } } - @PreDestroy - private void destroy() { + @Override + public void destroy() { if (channel != null) { try { channel.close(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index 6080baec26..8e293e6dff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -21,45 +21,56 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.CreateQueueRequest; -import com.amazonaws.services.sqs.model.QueueAttributeName; import org.thingsboard.server.queue.TbQueueAdmin; -import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class TbAwsSqsAdmin implements TbQueueAdmin { - private final TbAwsSqsSettings sqsSettings; - private final Map attributes = new HashMap<>(); - private final AWSStaticCredentialsProvider credProvider; + private final Map attributes; + private final AmazonSQS sqsClient; + private final Set queues; - public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings) { - this.sqsSettings = sqsSettings; + public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { + this.attributes = attributes; AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - this.credProvider = new AWSStaticCredentialsProvider(awsCredentials); + sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) + .withRegion(sqsSettings.getRegion()) + .build(); - attributes.put("FifoQueue", "true"); - attributes.put("ContentBasedDeduplication", "true"); - attributes.put(QueueAttributeName.VisibilityTimeout.toString(), sqsSettings.getVisibilityTimeout()); + queues = sqsClient + .listQueues() + .getQueueUrls() + .stream() + .map(this::getQueueNameFromUrl) + .collect(Collectors.toCollection(ConcurrentHashMap::newKeySet)); } @Override public void createTopicIfNotExists(String topic) { - AmazonSQS sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) - .withRegion(sqsSettings.getRegion()) - .build(); + String queueName = topic.replaceAll("\\.", "_") + ".fifo"; + if (queues.contains(queueName)) { + return; + } + final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); + String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); + queues.add(getQueueNameFromUrl(queueUrl)); + } + + private String getQueueNameFromUrl(String queueUrl) { + int delimiterIndex = queueUrl.lastIndexOf("/"); + return queueUrl.substring(delimiterIndex + 1); + } - final CreateQueueRequest createQueueRequest = - new CreateQueueRequest(topic.replaceAll("\\.", "_") + ".fifo") - .withAttributes(attributes); - try { - sqsClient.createQueue(createQueueRequest); - } finally { - if (sqsClient != null) { - sqsClient.shutdown(); - } + @Override + public void destroy() { + if (sqsClient != null) { + sqsClient.shutdown(); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java new file mode 100644 index 0000000000..70a9587bef --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.sqs; + +import com.amazonaws.services.sqs.model.QueueAttributeName; +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") +public class TbAwsSqsQueueAttributes { + @Value("${queue.aws-sqs.queue-properties.core}") + private String coreProperties; + @Value("${queue.aws-sqs.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.aws-sqs.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.aws-sqs.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.aws-sqs.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreAttributes; + @Getter + private Map ruleEngineAttributes; + @Getter + private Map transportApiAttributes; + @Getter + private Map notificationsAttributes; + @Getter + private Map jsExecutorAttributes; + + private final Map defaultAttributes = new HashMap<>(); + + @PostConstruct + private void init() { + defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); + defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); + + coreAttributes = getConfigs(coreProperties); + ruleEngineAttributes = getConfigs(ruleEngineProperties); + transportApiAttributes = getConfigs(transportApiProperties); + notificationsAttributes = getConfigs(notificationsProperties); + jsExecutorAttributes = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + validateAttributeName(key); + configs.put(key, value); + } + configs.putAll(defaultAttributes); + return configs; + } + + private void validateAttributeName(String key) { + QueueAttributeName.fromValue(key); + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java index 42a2f1e983..922a2b1062 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -39,6 +39,4 @@ public class TbAwsSqsSettings { @Value("${queue.aws_sqs.threads_per_topic}") private int threadsPerTopic; - @Value("${queue.aws_sqs.visibility_timeout}") - private String visibilityTimeout; } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index e2fb7e149a..78dc191e2c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -62,7 +62,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index dfd2c14be2..82b548a456 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -63,7 +63,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 81f0fb795d..e3faf9cb87 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -93,7 +93,13 @@ queue: secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - visibility_timeout: "${TB_QUEUE_AWS_SQS_VISIBILITY_TIMEOUT:30}" #In seconds. If messages wont commit in this time, messages will poll again + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" From 12973cecf1b05ca71d24d5545d60b74b6af3b176 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 12:51:05 +0300 Subject: [PATCH 078/196] added method destroy to admins and aws sqs factories --- .../provider/AwsSqsMonolithQueueFactory.java | 21 +++++++++++++++++++ .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++++++++++++++++++ .../AwsSqsTbRuleEngineQueueFactory.java | 18 ++++++++++++++++ .../provider/AwsSqsTransportQueueFactory.java | 12 +++++++++++ 4 files changed, 72 insertions(+) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 90fe643505..76ff04c238 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -39,6 +39,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -145,4 +147,23 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index e169e6502d..a7349091bd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -43,6 +43,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -134,4 +136,23 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 33f1d8c113..3056eced2c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -114,4 +116,20 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 644bc6b709..c9ba3ec15f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -38,6 +38,8 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -100,4 +102,14 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } From adae8aeb30631d292573833d58d621ef08606a5a Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Fri, 17 Apr 2020 16:55:17 +0300 Subject: [PATCH 079/196] Improved implementaton of ProcessingAttemptContext to avoid synchronizations --- .../queue/ProcessingAttemptContext.java | 21 +++----- .../queue/ProcessingAttemptContextTest.java | 51 +++++++++++++++++++ 2 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java index 2073ff8c21..aefb1697cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java @@ -27,11 +27,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class ProcessingAttemptContext { private final TbRuleEngineSubmitStrategy submitStrategy; + private final AtomicInteger pendingCount; private final CountDownLatch processingTimeoutLatch = new CountDownLatch(1); @Getter private final ConcurrentMap> pendingMap; @@ -45,6 +47,7 @@ public class ProcessingAttemptContext { public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { this.submitStrategy = submitStrategy; this.pendingMap = submitStrategy.getPendingMap(); + this.pendingCount = new AtomicInteger(pendingMap.size()); } public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { @@ -54,16 +57,12 @@ public class ProcessingAttemptContext { public void onSuccess(UUID id) { TbProtoQueueMsg msg; boolean empty = false; - synchronized (pendingMap) { - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingMap.isEmpty(); - } - } + msg = pendingMap.remove(id); if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; successMap.put(id, msg); + submitStrategy.onSuccess(id); } - submitStrategy.onSuccess(id); if (empty) { processingTimeoutLatch.countDown(); } @@ -72,13 +71,9 @@ public class ProcessingAttemptContext { public void onFailure(TenantId tenantId, UUID id, RuleEngineException e) { TbProtoQueueMsg msg; boolean empty = false; - synchronized (pendingMap) { - msg = pendingMap.remove(id); - if (msg != null) { - empty = pendingMap.isEmpty(); - } - } + msg = pendingMap.remove(id); if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; failedMap.put(id, msg); exceptionsMap.putIfAbsent(tenantId, e); } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java new file mode 100644 index 0000000000..6c3914abc8 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -0,0 +1,51 @@ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class ProcessingAttemptContextTest { + + @Test + public void testHighConcurrencyCase() throws InterruptedException { + TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + int msgCount = 1000; + int parallelCount = 5; + ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); + try { + ConcurrentMap> messages = new ConcurrentHashMap<>(); + for (int i = 0; i < msgCount; i++) { + messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); + } + when(strategyMock.getPendingMap()).thenReturn(messages); + ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + for (UUID uuid : messages.keySet()) { + for (int i = 0; i < parallelCount; i++) { + executorService.submit(() -> context.onSuccess(uuid)); + } + } + Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); + Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); + } finally { + executorService.shutdownNow(); + } + } +} From eae9d3c8e29ac276a1005c960951ac40535e6ad2 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Mon, 20 Apr 2020 10:55:25 +0300 Subject: [PATCH 080/196] Processing Atempt Context Improvement --- .../queue/ProcessingAttemptContextTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java index 6c3914abc8..2de2414f71 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; From b1f87206f3d930fbec7270ea018ba68a188315fc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 11:02:40 +0300 Subject: [PATCH 081/196] Minor improvements --- .../server/actors/ruleChain/DefaultTbContext.java | 3 +++ .../server/actors/ruleChain/RuleNodeActor.java | 2 ++ .../server/service/state/DefaultDeviceStateService.java | 8 ++++---- .../server/service/state/DeviceStateService.java | 2 +- .../common/transport/service/DefaultTransportService.java | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index b268236448..6a3994a36f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -192,6 +192,9 @@ class DefaultTbContext implements TbContext { @Override public void ack(TbMsg tbMsg) { + if (nodeCtx.getSelf().isDebugMode()) { + mainCtx.persistDebugOutput(nodeCtx.getTenantId(), nodeCtx.getSelf().getId(), tbMsg, "ACK", null); + } tbMsg.getCallback().onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java index 64431fdf33..abb98468bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; public class RuleNodeActor extends ComponentActor { @@ -54,6 +55,7 @@ public class RuleNodeActor extends ComponentActor> partitionedDevices = new ConcurrentHashMap<>(); - private ConcurrentMap deviceStates = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); - private ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap> partitionedDevices = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceStates = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastReportedActivity = new ConcurrentHashMap<>(); + private final ConcurrentMap deviceLastSavedActivity = new ConcurrentHashMap<>(); public DefaultDeviceStateService(TenantService tenantService, DeviceService deviceService, AttributesService attributesService, TimeseriesService tsService, diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 16c699b38a..1665a27b84 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -41,6 +41,6 @@ public interface DeviceStateService extends ApplicationListener callback.onSuccess(null)); } } From 341f0d14ab0ebe221b1ff2ea94bfa425980c03ab Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 12:05:27 +0300 Subject: [PATCH 082/196] Fixed tests inconsistency due to new queues --- .../mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java | 4 ++++ .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 5ba4dfcbae..485d618a01 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -109,6 +109,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", MqttQoS.AT_MOST_ONCE.value()); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -165,6 +167,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC client.subscribe("v1/devices/me/rpc/request/+", 1); client.setCallback(new TestMqttCallback(client, new CountDownLatch(1))); + Thread.sleep(2000); + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 122bef2e26..7e329d8b05 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -79,7 +79,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr String deviceId = savedDevice.getId().getId().toString(); - Thread.sleep(1000); + Thread.sleep(2000); List actualKeys = doGetAsync("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", List.class); Set actualKeySet = new HashSet<>(actualKeys); From 58d9c313a89d3929ad7c3d0e46ed287b91132355 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 20 Apr 2020 16:12:03 +0300 Subject: [PATCH 083/196] Fixed tests --- .../DefaultRuleEngineStatisticsService.java | 11 +- .../controller/BaseAssetControllerTest.java | 128 ++++++++++-------- .../BaseEntityViewControllerTest.java | 6 +- .../controller/ControllerNoSqlTestSuite.java | 7 + .../controller/ControllerSqlTestSuite.java | 7 + .../server/mqtt/MqttNoSqlTestSuite.java | 7 + .../server/mqtt/MqttSqlTestSuite.java | 7 + .../rules/RuleEngineNoSqlTestSuite.java | 7 + .../server/rules/RuleEngineSqlTestSuite.java | 7 + .../server/system/SystemNoSqlTestSuite.java | 7 + .../server/system/SystemSqlTestSuite.java | 7 + .../server/queue/memory/InMemoryStorage.java | 42 +++--- .../queue/memory/InMemoryTbQueueConsumer.java | 11 +- tools/pom.xml | 2 +- 14 files changed, 173 insertions(+), 83 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 3951703351..a506b87ab0 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -47,6 +47,7 @@ import java.util.stream.Collectors; @Slf4j public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { + public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; public static final FutureCallback CALLBACK = new FutureCallback() { @Override public void onSuccess(@Nullable Void result) { @@ -95,7 +96,13 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS }); ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + try { + tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + } catch (DataValidationException e2) { + if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { + throw e2; + } + } }); ruleEngineStats.reset(); } @@ -113,7 +120,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS asset = new Asset(); asset.setTenantId(tenantId); asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); - asset.setType("TbServiceQueue"); + asset.setType(TB_SERVICE_QUEUE); asset = assetService.saveAsset(asset); } assetId = asset.getId(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index 56c73b5e13..0422a85416 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.Collections; @@ -71,7 +72,7 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -111,26 +112,27 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(doPost("/api/asset", asset, Asset.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(doPost("/api/asset", asset, Asset.class)); } List assetTypes = doGetTyped("/api/asset/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(assetTypes); Assert.assertEquals(3, assetTypes.size()); @@ -146,10 +148,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - doDelete("/api/asset/"+savedAsset.getId().getId().toString()) + doDelete("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/asset/"+savedAsset.getId().getId().toString()) + doGet("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -244,16 +246,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) .andExpect(status().isOk()); } @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(doPost("/api/asset", asset, Asset.class)); } @@ -262,13 +264,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); + loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); + Collections.sort(assets, idComparator); Collections.sort(loadedAssets, idComparator); @@ -279,10 +284,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testFindTenantAssetsByName() throws Exception { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -290,10 +295,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -305,7 +310,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -321,7 +327,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -334,24 +341,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsTitle2, loadedAssetsTitle2); for (Asset asset : loadedAssetsTitle1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsTitle2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/tenant/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -361,10 +370,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -373,10 +382,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -388,7 +397,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -404,7 +414,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -417,24 +428,26 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(assetsType2, loadedAssetsType2); for (Asset asset : loadedAssetsType1) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Asset asset : loadedAssetsType2) { - doDelete("/api/asset/"+asset.getId().getId().toString()) + doDelete("/api/asset/" + asset.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -447,9 +460,9 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Asset asset = new Asset(); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = doPost("/api/asset", asset, Asset.class); assets.add(doPost("/api/customer/" + customerId.getId().toString() @@ -461,7 +474,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssets.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -483,10 +497,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -496,10 +510,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -513,7 +527,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -529,7 +544,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); loadedAssetsTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -548,7 +564,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title1); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -559,7 +576,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4, title2); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?", - new TypeReference>(){}, pageLink); + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -574,10 +592,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -588,10 +606,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Asset asset = new Asset(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -605,7 +623,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedAssetsType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -621,7 +640,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedAssetsType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -640,7 +660,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -651,7 +672,8 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/assets?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index e60f549610..420446e04d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -426,7 +426,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes message.setPayload(strKvs.getBytes()); client.publish("v1/devices/me/telemetry", message); Thread.sleep(1000); -// client.disconnect(); + client.disconnect(); } private void awaitConnected(MqttAsyncClient client, long ms) throws InterruptedException { @@ -463,13 +463,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes MqttConnectOptions options = new MqttConnectOptions(); options.setUserName(accessToken); client.connect(options); - Thread.sleep(3000); + awaitConnected(client, TimeUnit.SECONDS.toMillis(30)); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); client.publish("v1/devices/me/attributes", message); Thread.sleep(1000); - + client.disconnect(); return new HashSet<>(doGetAsync("/api/plugins/telemetry/DEVICE/" + viewDeviceId + "/keys/attributes", List.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java index 4ad41824a0..781c483fc5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.controller; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -37,4 +39,9 @@ public class ControllerNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false), new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 8dc0acff57..347eaee7eb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.controller; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class ControllerSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index 9c4030cff4..7360c5c506 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.mqtt; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -36,4 +38,9 @@ public class MqttNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 2863589ba1..70de3b27ca 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.mqtt; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -32,4 +34,9 @@ public class MqttSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java index e44c383452..fbab13147c 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java @@ -16,11 +16,13 @@ package org.thingsboard.server.rules; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -40,4 +42,9 @@ public class RuleEngineNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 5f930821f7..83ac5f7703 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.rules; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -33,4 +35,9 @@ public class RuleEngineSqlTestSuite { Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java index 41be7641c6..c4182db3ee 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java @@ -16,10 +16,12 @@ package org.thingsboard.server.system; import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomCassandraCQLUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -38,4 +40,9 @@ public class SystemNoSqlTestSuite { new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false), new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)), "cassandra-test.yaml", 30000l); + + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index b12d513ce0..6060a2cc2b 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.system; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.thingsboard.server.dao.CustomSqlUnit; +import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Arrays; @@ -35,4 +37,9 @@ public class SystemSqlTestSuite { "sql/drop-all-tables.sql", "sql-test.properties"); + @BeforeClass + public static void cleanupInMemStorage(){ + InMemoryStorage.getInstance().cleanup(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index f919df871e..e119d1433b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -30,11 +30,9 @@ import java.util.concurrent.TimeUnit; public final class InMemoryStorage { private static InMemoryStorage instance; private final ConcurrentHashMap> storage; - private volatile boolean stopped; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); - stopped = false; } public static InMemoryStorage getInstance() { @@ -52,33 +50,31 @@ public final class InMemoryStorage { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public List get(String topic, long durationInMillis) { + public List get(String topic, long durationInMillis) throws InterruptedException { if (storage.containsKey(topic)) { - try { - List entities; - T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); - if (first != null) { - entities = new ArrayList<>(); - entities.add(first); - List otherList = new ArrayList<>(); - storage.get(topic).drainTo(otherList, 999); - for (TbQueueMsg other : otherList) { - entities.add((T) other); - } - } else { - entities = Collections.emptyList(); - } - return entities; - } catch (InterruptedException e) { - if (!stopped) { - log.warn("Queue was interrupted", e); + List entities; + T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + if (first != null) { + entities = new ArrayList<>(); + entities.add(first); + List otherList = new ArrayList<>(); + storage.get(topic).drainTo(otherList, 999); + for (TbQueueMsg other : otherList) { + entities.add((T) other); } + } else { + entities = Collections.emptyList(); } + return entities; } return Collections.emptyList(); } - public void stop() { - stopped = true; + /** + * Used primarily for testing. + */ + public void cleanup() { + storage.clear(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 2b81839540..3920f143f1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -66,7 +66,16 @@ public class InMemoryTbQueueConsumer implements TbQueueCon if (subscribed) { List messages = partitions .stream() - .map(tpi -> storage.get(tpi.getFullTopicName(), durationInMillis)) + .map(tpi -> { + try { + return storage.get(tpi.getFullTopicName(), durationInMillis); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Queue was interrupted.", e); + } + return Collections.emptyList(); + } + }) .flatMap(List::stream) .map(msg -> (T) msg).collect(Collectors.toList()); if (messages.size() > 0) { diff --git a/tools/pom.xml b/tools/pom.xml index 64027c478e..1960d330d0 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ org.apache.cassandra cassandra-all - 3.11.4 + 3.11.6 com.datastax.cassandra From 149ec2804b410ab7e099154f139f55c01e6ae696 Mon Sep 17 00:00:00 2001 From: Ilya Barkov Date: Mon, 20 Apr 2020 16:20:33 +0300 Subject: [PATCH 084/196] Feature/gateway remote configuration (#2627) * new widget for gateway with the last telemetry type * add isStateForm for the last telemetry type * add config update save * update thingboard yaml * remove update thingboard yaml * add new widget to gateway_widgets.json * delete state gateway * delete state gateway * Revert "delete state gateway" This reverts commit d8024ead * change yml * edit name isReadOnly * fix bug isReadOnly * [Gateway remote configuration] Changed default logs.conf file Co-authored-by: nick --- .../widget_bundles/gateway_widgets.json | 32 ++++++++ ui/src/app/common/types.constant.js | 18 ++++- .../gateway/gateway-config.directive.js | 7 +- .../gateway/gateway-config.tpl.html | 12 +-- .../gateway/gateway-form.directive.js | 80 ++++++++++++------- .../components/gateway/gateway-form.tpl.html | 63 ++++++++------- ui/src/app/locale/locale.constant-en_US.json | 7 +- 7 files changed, 153 insertions(+), 66 deletions(-) diff --git a/application/src/main/data/json/system/widget_bundles/gateway_widgets.json b/application/src/main/data/json/system/widget_bundles/gateway_widgets.json index 0158b38fe9..7d5dd46086 100644 --- a/application/src/main/data/json/system/widget_bundles/gateway_widgets.json +++ b/application/src/main/data/json/system/widget_bundles/gateway_widgets.json @@ -5,6 +5,38 @@ "image": null }, "widgetTypes": [ + { + "alias": "attributes_card", + "name": "Gateway events", + "descriptor": { + "type": "latest", + "sizeX": 7.5, + "sizeY": 8, + "resources": [], + "templateHtml": "", + "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "controllerScript": "let types;\nlet eventsReg = \"eventsReg\";\n\nself.onInit = function() {\n \n self.ctx.datasourceTitleCells = [];\n self.ctx.valueCells = [];\n self.ctx.labelCells = [];\n types = self.ctx.$scope.$injector.get('types');\n \n if (self.ctx.datasources.length && self.ctx.datasources[0].type === types.datasourceType.entity) {\n getDatasourceKeys(self.ctx.datasources[0]);\n } else {\n processDatasources(self.ctx.datasources);\n }\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.valueCells.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData && cellData.data && cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length -\n 1];\n var value = tvPair[1];\n var textValue;\n //toDo -> + IsNumber\n \n if (isNumber(value)) {\n var decimals = self.ctx.decimals;\n var units = self.ctx.units;\n if (cellData.dataKey.decimals || cellData.dataKey.decimals === 0) {\n decimals = cellData.dataKey.decimals;\n }\n if (cellData.dataKey.units) {\n units = cellData.dataKey.units;\n }\n txtValue = self.ctx.utils.formatValue(value, decimals, units, false);\n }\n else {\n txtValue = value;\n }\n self.ctx.valueCells[i].html(txtValue);\n }\n }\n \n function isNumber(n) {\n return !isNaN(parseFloat(n)) && isFinite(n);\n }\n}\n\nself.onResize = function() {\n var datasourceTitleFontSize = self.ctx.height/8;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n datasourceTitleFontSize = self.ctx.width/12;\n }\n datasourceTitleFontSize = Math.min(datasourceTitleFontSize, 20);\n for (var i = 0; i < self.ctx.datasourceTitleCells.length; i++) {\n self.ctx.datasourceTitleCells[i].css('font-size', datasourceTitleFontSize+'px');\n }\n var valueFontSize = self.ctx.height/9;\n var labelFontSize = self.ctx.height/9;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n valueFontSize = self.ctx.width/15;\n labelFontSize = self.ctx.width/15;\n }\n valueFontSize = Math.min(valueFontSize, 18);\n labelFontSize = Math.min(labelFontSize, 18);\n\n for (i = 0; i < self.ctx.valueCells; i++) {\n self.ctx.valueCells[i].css('font-size', valueFontSize+'px');\n self.ctx.valueCells[i].css('height', valueFontSize*2.5+'px');\n self.ctx.valueCells[i].css('padding', '0px ' + valueFontSize + 'px');\n self.ctx.labelCells[i].css('font-size', labelFontSize+'px');\n self.ctx.labelCells[i].css('height', labelFontSize*2.5+'px');\n self.ctx.labelCells[i].css('padding', '0px ' + labelFontSize + 'px');\n } \n}\n\nfunction processDatasources(datasources) {\n var i = 0;\n var tbDatasource = datasources[i];\n var datasourceId = 'tbDatasource' + i;\n self.ctx.$container.append(\n \"
\"\n );\n\n var datasourceContainer = $('#' + datasourceId,\n self.ctx.$container);\n\n datasourceContainer.append(\n \"
\" +\n tbDatasource.name + \"
\"\n );\n \n var datasourceTitleCell = $('.tbDatasource-title', datasourceContainer);\n self.ctx.datasourceTitleCells.push(datasourceTitleCell);\n \n var tableId = 'table' + i;\n datasourceContainer.append(\n \"
dashboard.state-name dashboard.state-id
\"\n );\n var table = $('#' + tableId, self.ctx.$container);\n\n for (var a = 0; a < tbDatasource.dataKeys.length; a++) {\n var dataKey = tbDatasource.dataKeys[a];\n var labelCellId = 'labelCell' + a;\n var cellId = 'cell' + a;\n table.append(\"\" + dataKey.label +\n \"\");\n var labelCell = $('#' + labelCellId, table);\n self.ctx.labelCells.push(labelCell);\n var valueCell = $('#' + cellId, table);\n self.ctx.valueCells.push(valueCell);\n }\n self.onResize();\n}\n\nfunction getDatasourceKeys (datasource) {\n let attributeService = self.ctx.$scope.$injector.get('attributeService');\n if (datasource.entityId && datasource.entityType) {\n attributeService.getEntityKeys(datasource.entityType, datasource.entityId, \"\" , types.dataKeyType.timeseries).then(\n function(data){\n if (data.length) {\n subscribeForKeys (datasource, data);\n }\n });\n }\n}\n\nfunction subscribeForKeys (datasource, data) {\n let eventsRegVals = self.ctx.settings[eventsReg];\n if (eventsRegVals && eventsRegVals.length > 0) {\n var dataKeys = [];\n data.sort();\n data.forEach(dataValue => {eventsRegVals.forEach(event => {\n if (dataValue.toLowerCase().includes(event.toLowerCase())) {\n var dataKey = {\n type: types.dataKeyType.timeseries,\n name: dataValue,\n label: dataValue,\n settings: {},\n _hash: Math.random()\n };\n dataKeys.push(dataKey);\n }\n })});\n\n if (dataKeys.length) {\n updateSubscription (datasource, dataKeys);\n }\n }\n}\n\nfunction updateSubscription (datasource, dataKeys) {\n var datasources = [\n {\n type: types.datasourceType.entity,\n name: datasource.aliasName,\n aliasName: datasource.aliasName,\n entityAliasId: datasource.entityAliasId,\n dataKeys: dataKeys\n }\n ];\n \n var subscriptionOptions = {\n datasources: datasources,\n useDashboardTimewindow: false,\n type: types.widgetType.latest.value,\n callbacks: {\n onDataUpdated: (subscription) => {\n self.ctx.data = subscription.data;\n self.onDataUpdated();\n }\n }\n };\n \n processDatasources(datasources);\n self.ctx.subscriptionApi.createSubscription(subscriptionOptions, true).then(\n (subscription) => {\n self.ctx.defaultSubscription = subscription;\n }\n );\n}\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\n dataKeysOptional: true\n };\n}\n\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"GatewayEventsForm\",\n \"properties\": {\n \"eventsTitle\": {\n \"title\": \"Gateway events form title\",\n \"type\": \"string\",\n \"default\": \"Gateway Events Form\"\n },\n \"eventsReg\": {\n \"title\": \"Events filten.\",\n \"type\": \"array\",\n \"items\": {\n \"title\": \"Event key contains\",\n \"type\": \"string\"\n }\n }\n }\n },\n \"form\": [\n \"eventsTitle\",\n \"eventsReg\"\n ]\n}", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Function Math.round\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.826503672916844,\"funcBody\":\"return Math.round(1000*Math.sin(time/5000));\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"eventsTitle\":\"Gateway Events Form\",\"eventsReg\":[]},\"title\":\"Gateway events\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":true,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" + } + }, + { + "alias": "config_form_latest", + "name": "Gateway configuration (Single device)", + "descriptor": { + "type": "latest", + "sizeX": 7.5, + "sizeY": 9, + "resources": [], + "templateHtml": "\n", + "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}", + "controllerScript": "self.onInit = function() {\n var scope = self.ctx.$scope;\n var id = self.ctx.$scope.$injector.get('utils').guid();\n scope.formId = \"form-\"+id;\n scope.ctx = self.ctx;\n scope.isStateForm = true;\n}\n\nself.onResize = function() {\n self.ctx.$scope.$broadcast('gateway-form-resize', self.ctx.$scope.formId);\n}\n\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\t\t\n dataKeysOptional: true\t\t\n };\n}\n\n", + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"GatewayConfigForm\",\n \"properties\": {\n \"gatewayTitle\": {\n \"title\": \"Gateway form\",\n \"type\": \"string\",\n \"default\": \"Gateway configuration (Single device)\"\n },\n \"readOnly\": {\n \"title\": \"Read Only\",\n \"type\": \"boolean\",\n \"default\": false\n }\n }\n },\n \"form\": [\n \"gatewayTitle\",\n \"readOnly\"\n ]\n}\n", + "dataKeySettingsSchema": "{}\n", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Gateway configuration (Single device)\"}" + } + }, { "alias": "extension_configuration_widget", "name": "Extensions table", diff --git a/ui/src/app/common/types.constant.js b/ui/src/app/common/types.constant.js index aff3821573..8285430c56 100644 --- a/ui/src/app/common/types.constant.js +++ b/ui/src/app/common/types.constant.js @@ -606,13 +606,29 @@ export default angular.module('thingsboard.types', []) value: "modbus", name: "Modbus" }, - opc_ua: { + opcua: { value: "opcua", name: "OPC-UA" }, ble: { value: "ble", name: "BLE" + }, + request: { + value: "request", + name: "Request" + }, + can: { + value: "can", + name: "CAN" + }, + bacnet: { + value: "bacnet", + name: "BACnet" + }, + custom: { + value: "custom", + name: "Custom" } }, gatewayLogLevel: { diff --git a/ui/src/app/components/gateway/gateway-config.directive.js b/ui/src/app/components/gateway/gateway-config.directive.js index 120a6d3e15..19e7f554b9 100644 --- a/ui/src/app/components/gateway/gateway-config.directive.js +++ b/ui/src/app/components/gateway/gateway-config.directive.js @@ -37,7 +37,8 @@ function GatewayConfig() { disabled: '=ngDisabled', gatewayConfig: '=', changeAlignment: '=', - theForm: '=' + theForm: '=', + isReadOnly: '=' }, controller: GatewayConfigController, controllerAs: 'vm', @@ -83,6 +84,10 @@ function GatewayConfigController($scope, $document, $mdDialog, $mdUtil, $window, multiple: true, }).then(function (config) { if (config && index > -1) { + console.log(config); //eslint-disable-line + if (!angular.equals(vm.gatewayConfig[index].config, config)) { + $scope.gatewayConfiguration.$setDirty(); + } vm.gatewayConfig[index].config = config; } }); diff --git a/ui/src/app/components/gateway/gateway-config.tpl.html b/ui/src/app/components/gateway/gateway-config.tpl.html index eef7c737e4..e4525b2bf2 100644 --- a/ui/src/app/components/gateway/gateway-config.tpl.html +++ b/ui/src/app/components/gateway/gateway-config.tpl.html @@ -18,7 +18,7 @@
-
@@ -26,7 +26,7 @@ ng-class="{'gateway-config-row-vertical': vm.changeAlignment}"> - @@ -39,7 +39,7 @@
-
@@ -49,7 +49,7 @@
- more_horiz @@ -57,7 +57,7 @@ {{ 'gateway.update-config' | translate }} - close @@ -70,7 +70,7 @@ layout-align="center center" ng-class="{'disabled': vm.disabled}" class="no-data-found" translate>{{'gateway.no-connectors'}}
- {{ 'gateway.connector-add' | translate }} diff --git a/ui/src/app/components/gateway/gateway-form.directive.js b/ui/src/app/components/gateway/gateway-form.directive.js index 5efa2ee121..436bc6bcd5 100644 --- a/ui/src/app/components/gateway/gateway-form.directive.js +++ b/ui/src/app/components/gateway/gateway-form.directive.js @@ -31,11 +31,15 @@ function GatewayForm() { scope: true, bindToController: { formId: '=', - ctx: '=' + ctx: '=', + isStateForm: '=', + deviceName: '=', + isReadOnly: '=', + isState: '=' }, controller: GatewayFormController, controllerAs: 'vm', - templateUrl: gatewayFormTemplate + templateUrl: gatewayFormTemplate, }; } @@ -47,10 +51,11 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, const configurationAttribute = "configuration"; const remoteLoggingLevelAttribute = "RemoteLoggingLevel"; - const templateLogsConfig = '[loggers]}}keys=root, service, connector, converter, tb_connection, storage, extension}}[handlers]}}keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler}}[formatters]}}keys=LogFormatter}}[logger_root]}}level=ERROR}}handlers=consoleHandler}}[logger_connector]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=connector}}[logger_storage]}}level={ERROR}}}handlers=storageHandler}}formatter=LogFormatter}}qualname=storage}}[logger_tb_connection]}}level={ERROR}}}handlers=tb_connectionHandler}}formatter=LogFormatter}}qualname=tb_connection}}[logger_service]}}level={ERROR}}}handlers=serviceHandler}}formatter=LogFormatter}}qualname=service}}[logger_converter]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=converter}}[logger_extension]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=extension}}[handler_consoleHandler]}}class=StreamHandler}}level={ERROR}}}formatter=LogFormatter}}args=(sys.stdout,)}}[handler_connectorHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}connector.log", "d", 1, 7,)}}[handler_storageHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}storage.log", "d", 1, 7,)}}[handler_serviceHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}service.log", "d", 1, 7,)}}[handler_converterHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}converter.log", "d", 1, 3,)}}[handler_extensionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}extension.log", "d", 1, 3,)}}[handler_tb_connectionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}tb_connection.log", "d", 1, 3,)}}[formatter_LogFormatter]}}format="%(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s" }}datefmt="%Y-%m-%d %H:%M:%S"'; + const templateLogsConfig = '[loggers]}}keys=root, service, connector, converter, tb_connection, storage, extension}}[handlers]}}keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler}}[formatters]}}keys=LogFormatter}}[logger_root]}}level=ERROR}}handlers=consoleHandler}}[logger_connector]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=connector}}[logger_storage]}}level={ERROR}}}handlers=storageHandler}}formatter=LogFormatter}}qualname=storage}}[logger_tb_connection]}}level={ERROR}}}handlers=tb_connectionHandler}}formatter=LogFormatter}}qualname=tb_connection}}[logger_service]}}level={ERROR}}}handlers=serviceHandler}}formatter=LogFormatter}}qualname=service}}[logger_converter]}}level={ERROR}}}handlers=converterHandler}}formatter=LogFormatter}}qualname=converter}}[logger_extension]}}level={ERROR}}}handlers=connectorHandler}}formatter=LogFormatter}}qualname=extension}}[handler_consoleHandler]}}class=StreamHandler}}level={ERROR}}}formatter=LogFormatter}}args=(sys.stdout,)}}[handler_connectorHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}connector.log", "d", 1, 7,)}}[handler_storageHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}storage.log", "d", 1, 7,)}}[handler_serviceHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}service.log", "d", 1, 7,)}}[handler_converterHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}converter.log", "d", 1, 3,)}}[handler_extensionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}extension.log", "d", 1, 3,)}}[handler_tb_connectionHandler]}}level={ERROR}}}class=logging.handlers.TimedRotatingFileHandler}}formatter=LogFormatter}}args=("{./logs/}tb_connection.log", "d", 1, 3,)}}[formatter_LogFormatter]}}format="%(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s" }}datefmt="%Y-%m-%d %H:%M:%S"'; vm.types = types; - + vm.deviceNameForm = (vm.deviceName) ? vm.deviceName : null; + vm.idForm = Math.random().toString(36).replace(/^0\.[0-9]*/, ''); vm.configurations = { gateway: '', host: $document[0].domain, @@ -74,7 +79,6 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, let archiveFileName = ''; let gatewayNameExists = ''; let successfulSaved = ''; - vm.securityTypes = [{ name: 'gateway.security-types.access-token', value: 'accessToken' @@ -92,13 +96,18 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, }]; $scope.$watch('vm.ctx', function () { - if (vm.ctx ) { + if (vm.ctx) { + vm.isStateForm = $scope.isStateForm; vm.settings = vm.ctx.settings; vm.widgetConfig = vm.ctx.widgetConfig; - initializeConfig(); + if (vm.ctx.datasources && vm.ctx.datasources.length) { + vm.deviceNameForm = vm.ctx.datasources[0].name; + } } + initializeConfig(); }); + $scope.$on('gateway-form-resize', function (event, formId) { if (vm.formId == formId) { updateWidgetDisplaying(); @@ -106,21 +115,34 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, }); function updateWidgetDisplaying() { - vm.changeAlignment = (vm.ctx.$container[0].offsetWidth <= 425); + if (vm.ctx) { + vm.changeAlignment = (vm.ctx.$container[0].offsetWidth <= 425); + } } function initWidgetSettings() { let widgetTitle; - if (vm.settings.widgetTitle && vm.settings.widgetTitle.length) { - widgetTitle = utils.customTranslation(vm.settings.widgetTitle, vm.settings.widgetTitle); + if (vm.settings) { + vm.isReadOnlyForm = (vm.settings.readOnly) ? vm.settings.readOnly : false; + if (vm.settings.gatewayTitle && vm.settings.gatewayTitle.length) { + widgetTitle = utils.customTranslation(vm.settings.gatewayTitle, vm.settings.gatewayTitle); + } } else { + vm.isReadOnlyForm = false; widgetTitle = $translate.instant('gateway.gateway'); } - vm.ctx.widgetTitle = widgetTitle; + if (vm.ctx) { + vm.ctx.widgetTitle = widgetTitle; + } - archiveFileName = vm.settings.archiveFileName && vm.settings.archiveFileName.length ? vm.settings.archiveFileName : 'gatewayConfiguration'; - gatewayNameExists = utils.customTranslation(vm.settings.gatewayNameExists, vm.settings.gatewayNameExists) || $translate.instant('gateway.gateway-exists'); - successfulSaved = utils.customTranslation(vm.settings.successfulSave, vm.settings.successfulSave) || $translate.instant('gateway.gateway-saved'); + archiveFileName = vm.settings && vm.settings.archiveFileName && vm.settings.archiveFileName.length ? vm.settings.archiveFileName : 'gatewayConfiguration'; + if (vm.settings) { + gatewayNameExists = utils.customTranslation(vm.settings.deviceNameExist, vm.settings.deviceNameExist) || $translate.instant('gateway.gateway-exists'); + successfulSaved = utils.customTranslation(vm.settings.successfulSave, vm.settings.successfulSave) || $translate.instant('gateway.gateway-saved'); + } else { + gatewayNameExists = $translate.instant('gateway.gateway-exists'); + successfulSaved = $translate.instant('gateway.gateway-saved'); + } } function initializeConfig() { @@ -129,6 +151,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, getGatewaysList(true); } + vm.getAccessToken = (deviceId) => { if (deviceId.id) { getDeviceCredentials(deviceId.id); @@ -152,15 +175,15 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, deviceService.findByName(deviceObj.name, {ignoreErrors: true}) .then( function () { - toast.showError(gatewayNameExists, angular.element('.gateway-form'),'top left'); + toast.showError(gatewayNameExists, angular.element('.gateway-form'), 'top left'); }, function () { - if(vm.settings.gatewayType && vm.settings.gatewayType.length){ + if (vm.settings.gatewayType && vm.settings.gatewayType.length) { deviceObj.type = vm.settings.gatewayType; } deviceService.saveDevice(deviceObj).then( (device) => { - getDeviceCredentials(device.id.id).then(() =>{ + getDeviceCredentials(device.id.id).then(() => { getGatewaysList(); }); } @@ -173,8 +196,8 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, saveAttribute(configurationAttribute, $window.btoa(angular.toJson(getGatewayConfigJSON())), types.attributesScope.shared.value), saveAttribute(configurationDraftsAttribute, $window.btoa(angular.toJson(getDraftConnectorJSON())), types.attributesScope.server.value), saveAttribute(remoteLoggingLevelAttribute, vm.configurations.remoteLoggingLevel.toUpperCase(), types.attributesScope.shared.value) - ]).then(() =>{ - toast.showSuccess(successfulSaved, 2000, angular.element('.gateway-form'),'top left'); + ]).then(() => { + toast.showSuccess(successfulSaved, 2000, angular.element('.gateway-form'), 'top left'); }) }; @@ -238,7 +261,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, config += ' max_records_per_file: ' + vm.configurations.maxRecordsCount + '\n'; } config += 'connectors:\n'; - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (vm.configurations.connectors[i].enabled) { config += ' -\n'; config += ' name: ' + vm.configurations.connectors[i].name + '\n'; @@ -250,7 +273,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, } function generateConfigConnectorFiles(fileZipAdd) { - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (vm.configurations.connectors[i].enabled) { fileZipAdd[generateFileName(vm.configurations.connectors[i].name)] = angular.toJson(vm.configurations.connectors[i].config); } @@ -276,7 +299,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, function gatewayMainConfigJSON() { let configuration = {}; - + let thingsBoard = {}; thingsBoard.host = vm.configurations.host; thingsBoard.remoteConfiguration = vm.configurations.remoteConfiguration; @@ -325,10 +348,10 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, } function gatewayConnectorConfigJSON(gatewayConfiguration) { - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (vm.configurations.connectors[i].enabled) { let typeConnector = vm.configurations.connectors[i].configType; - if(!angular.isArray(gatewayConfiguration[typeConnector])){ + if (!angular.isArray(gatewayConfiguration[typeConnector])) { gatewayConfiguration[typeConnector] = []; } @@ -343,7 +366,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, function getDraftConnectorJSON() { let draftConnector = {}; - for(let i = 0; i < vm.configurations.connectors.length; i++){ + for (let i = 0; i < vm.configurations.connectors.length; i++) { if (!vm.configurations.connectors[i].enabled) { let connector = { connector: vm.configurations.connectors[i].configType, @@ -362,7 +385,10 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, const device = devices[i]; if (device.additionalInfo !== null && device.additionalInfo.gateway === true) { vm.gateways.push(device); - if (firstInit && vm.gateways.length && device.name === vm.gateways[0].name) { + if (vm.deviceNameForm && firstInit && vm.gateways.length && device.name === vm.deviceNameForm) { + vm.configurations.gateway = device; + vm.getAccessToken(device.id); + } else if (firstInit && vm.gateways.length && device.name === vm.gateways[0].name) { vm.configurations.gateway = device; vm.getAccessToken(device.id); } @@ -385,7 +411,7 @@ function GatewayFormController($scope, $injector, $document, $mdExpansionPanel, for (let connectorType in keyValue) { let name = "No name"; if (Object.prototype.hasOwnProperty.call(keyValue[connectorType], 'name')) { - name = keyValue[connectorType].name ; + name = keyValue[connectorType].name; } let connector = { enabled: true, diff --git a/ui/src/app/components/gateway/gateway-form.tpl.html b/ui/src/app/components/gateway/gateway-form.tpl.html index abe030d543..c8d44d1fcb 100644 --- a/ui/src/app/components/gateway/gateway-form.tpl.html +++ b/ui/src/app/components/gateway/gateway-form.tpl.html @@ -17,30 +17,36 @@ --> - - + +
{{ 'gateway.thingsboard' | translate | uppercase }}
 - +
{{ 'gateway.thingsboard' | translate | uppercase }}
 - + + + + - + {{securityType.name | translate}} @@ -50,14 +56,14 @@ ng-class="{'gateway-config-row-vertical': vm.changeAlignment}"> - +
gateway.thingsboard-host-required
 -
gateway.thingsboard-port-required
@@ -70,18 +76,18 @@
- + - + - +
- {{ 'gateway.remote' | translate }} @@ -90,7 +96,7 @@ ng-class="{'gateway-config-row-vertical': vm.changeAlignment}"> - + {{logLevel}} @@ -99,7 +105,7 @@ -
gateway.path-logs-required
@@ -109,14 +115,14 @@ - +
{{ 'gateway.storage' | translate | uppercase }}
 - +
{{ 'gateway.storage' | translate | uppercase }}
@@ -124,18 +130,17 @@ - + {{storageType.name | translate}} -
-
gateway.storage-pack-size-required
@@ -143,10 +148,9 @@
gateway.storage-pack-size-pattern
 - -
gateway.storage-max-records-required
@@ -155,13 +159,12 @@
-
-
gateway.storage-max-files-required
@@ -169,10 +172,9 @@
gateway.storage-max-files-pattern
 - -
gateway.storage-path-required
@@ -182,14 +184,14 @@ - +
{{ 'gateway.connectors' | translate | uppercase }}
 - +
{{ 'gateway.connectors' | translate | uppercase }}
@@ -198,13 +200,14 @@ + change-alignment="vm.changeAlignment" + is-read-only="vm.isReadOnlyForm"> -
+
{{'gateway.download-tip' | translate }} - {{'action.save' | translate }} - {{'gateway.save-tip' | translate }} + {{'gateway.save-tip' | translate }} +
diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index d69778a787..8c7a89580d 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1201,7 +1201,12 @@ "tls-path-private-key": "Path to private key on gateway", "toggle-fullscreen": "Toggle fullscreen", "transformer-json-config": "Configuration JSON*", - "update-config": "Add/update configuration JSON" + "update-config": "Add/update configuration JSON", + "state-title": "Gateway state", + "show-config-tip": "Show gateway configuration", + "title-show-config": "Show gateway configuration", + "read-only": "Read only", + "read-write": "" }, "grid": { "delete-item-title": "Are you sure you want to delete this item?", From 1b9df18c4592ea16d35b0bf0eace8e1f20a2ca43 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 20:30:56 +0300 Subject: [PATCH 085/196] created TbPubSubSubscriptionSettings --- .../src/main/resources/thingsboard.yml | 7 +- .../provider/PubSubMonolithQueueFactory.java | 40 +++++++---- .../provider/PubSubTbCoreQueueFactory.java | 36 ++++++---- .../PubSubTbRuleEngineQueueFactory.java | 32 ++++++--- .../provider/PubSubTransportQueueFactory.java | 36 ++++++---- .../server/queue/pubsub/TbPubSubAdmin.java | 35 +++++++-- .../server/queue/pubsub/TbPubSubSettings.java | 3 - .../pubsub/TbPubSubSubscriptionSettings.java | 71 +++++++++++++++++++ .../src/main/resources/tb-coap-transport.yml | 7 +- .../src/main/resources/tb-http-transport.yml | 7 +- .../src/main/resources/tb-mqtt-transport.yml | 7 +- 11 files changed, 219 insertions(+), 62 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e1296275cd..0cae3252ff 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -543,9 +543,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index b1afc61dbd..ba806c5daf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -38,6 +38,7 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -53,88 +54,99 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.admin = new TbPubSubAdmin(pubSubSettings); this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getResponsesTopic()); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index edfcdc3d8e..eaffce241f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; @@ -47,71 +49,79 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueAdmin admin, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; - this.admin = admin; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, coreSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic(), + return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 101f335536..3206f03250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -32,9 +32,11 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -46,59 +48,67 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; + public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueAdmin admin, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; - this.admin = admin; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic(), + return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index f15faa11da..2a5de01f86 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -25,12 +25,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -38,6 +34,11 @@ import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -50,33 +51,42 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings) { + TbQueueTransportNotificationSettings transportNotificationSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; - this.admin = new TbPubSubAdmin(pubSubSettings); + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); + this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, transportApiSettings.getRequestsTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producer); templateBuilder.responseTemplate(consumer); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -87,17 +97,17 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(admin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index cd575818f2..f3b6d3f10c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -19,32 +19,37 @@ import com.google.cloud.pubsub.v1.SubscriptionAdminClient; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.pubsub.v1.TopicAdminSettings; +import com.google.protobuf.Duration; import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectSubscriptionName; import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PushConfig; import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @Slf4j public class TbPubSubAdmin implements TbQueueAdmin { + private static final String ACK_DEADLINE = "ackDeadlineInSec"; + private static final String MESSAGE_RETENTION = "messageRetentionInSec"; private final TbPubSubSettings pubSubSettings; private final SubscriptionAdminSettings subscriptionAdminSettings; private final TopicAdminSettings topicAdminSettings; private final Set topicSet = ConcurrentHashMap.newKeySet(); private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); + private final Map subscriptionProperties; - public TbPubSubAdmin(TbPubSubSettings pubSubSettings) { + public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { this.pubSubSettings = pubSubSettings; + this.subscriptionProperties = subscriptionSettings; try { topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); @@ -149,8 +154,15 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } - subscriptionAdminClient.createSubscription( - subscriptionName, topicName, PushConfig.getDefaultInstance(), pubSubSettings.getAckDeadline()).getName(); + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); + + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); + + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); subscriptionSet.add(subscriptionName.toString()); log.info("Created new subscription: [{}]", subscriptionName.toString()); } catch (IOException e) { @@ -159,4 +171,19 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } + private void setAckDeadline(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(ACK_DEADLINE)) { + builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); + } + } + + private void setMessageRetention(Subscription.Builder builder) { + if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { + Duration duration = Duration + .newBuilder() + .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) + .build(); + builder.setMessageRetentionDuration(duration); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java index 851a2b3245..d6c6d0e271 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -40,9 +40,6 @@ public class TbPubSubSettings { @Value("${queue.pubsub.service_account}") private String serviceAccount; - @Value("${queue.pubsub.ack_deadline}") - private int ackDeadline; - @Value("${queue.pubsub.max_msg_size}") private int maxMsgSize; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java new file mode 100644 index 0000000000..e98f51ec7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.pubsub; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +public class TbPubSubSubscriptionSettings { + @Value("${queue.pubsub.queue-properties.core}") + private String coreProperties; + @Value("${queue.pubsub.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.pubsub.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.pubsub.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.pubsub.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreSettings; + @Getter + private Map ruleEngineSettings; + @Getter + private Map transportApiSettings; + @Getter + private Map notificationsSettings; + @Getter + private Map jsExecutorSettings; + + @PostConstruct + private void init() { + coreSettings = getSettings(coreProperties); + ruleEngineSettings = getSettings(ruleEngineProperties); + transportApiSettings = getSettings(transportApiProperties); + notificationsSettings = getSettings(notificationsProperties); + jsExecutorSettings = getSettings(jsExecutorProperties); + } + + private Map getSettings(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 78dc191e2c..c3e646c77c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -72,9 +72,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 82b548a456..0d48bda2b9 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -73,9 +73,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e3faf9cb87..4249757db1 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -103,9 +103,14 @@ queue: pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - ack_deadline: "${TB_QUEUE_PUBSUB_ACK_DEADLINE:30}" #In seconds. If messages wont commit in this time, messages will poll again max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" From 2ad4ddf1fbe0c1350e31a70e369fe96c0085bb84 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 17 Apr 2020 21:05:12 +0300 Subject: [PATCH 086/196] pubsub improvements --- .../provider/PubSubMonolithQueueFactory.java | 21 ++++ .../provider/PubSubTbCoreQueueFactory.java | 18 ++++ .../PubSubTbRuleEngineQueueFactory.java | 18 ++++ .../provider/PubSubTransportQueueFactory.java | 18 ++++ .../server/queue/pubsub/TbPubSubAdmin.java | 101 +++++++++--------- 5 files changed, 124 insertions(+), 52 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index ba806c5daf..af0b276c1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -45,6 +45,8 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -153,4 +155,23 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index eaffce241f..0a23d58e46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -42,6 +42,8 @@ import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -128,4 +130,20 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 3206f03250..79501130ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -41,6 +41,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -117,4 +119,20 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 2a5de01f86..7e859e02ab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -111,4 +113,20 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index f3b6d3f10c..1241370d05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -40,9 +40,10 @@ public class TbPubSubAdmin implements TbQueueAdmin { private static final String ACK_DEADLINE = "ackDeadlineInSec"; private static final String MESSAGE_RETENTION = "messageRetentionInSec"; + private final TopicAdminClient topicAdminClient; + private final SubscriptionAdminClient subscriptionAdminClient; + private final TbPubSubSettings pubSubSettings; - private final SubscriptionAdminSettings subscriptionAdminSettings; - private final TopicAdminSettings topicAdminSettings; private final Set topicSet = ConcurrentHashMap.newKeySet(); private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); private final Map subscriptionProperties; @@ -51,6 +52,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { this.pubSubSettings = pubSubSettings; this.subscriptionProperties = subscriptionSettings; + TopicAdminSettings topicAdminSettings; try { topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); } catch (IOException e) { @@ -58,6 +60,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to create TopicAdminSettings."); } + SubscriptionAdminSettings subscriptionAdminSettings; try { subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); } catch (IOException e) { @@ -65,7 +68,9 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to create SubscriptionAdminSettings."); } - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + try { + topicAdminClient = TopicAdminClient.create(topicAdminSettings); + ListTopicsRequest listTopicsRequest = ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); @@ -77,7 +82,8 @@ public class TbPubSubAdmin implements TbQueueAdmin { throw new RuntimeException("Failed to get topics.", e); } - try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { + try { + subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); ListSubscriptionsRequest listSubscriptionsRequest = ListSubscriptionsRequest.newBuilder() @@ -104,31 +110,21 @@ public class TbPubSubAdmin implements TbQueueAdmin { return; } - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - if (topic.getName().contains(topicName.toString())) { - topicSet.add(topic.getName()); - createSubscriptionIfNotExists(partition, topicName); - return; - } + ListTopicsRequest listTopicsRequest = + ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); + TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); + for (Topic topic : response.iterateAll()) { + if (topic.getName().contains(topicName.toString())) { + topicSet.add(topic.getName()); + createSubscriptionIfNotExists(partition, topicName); + return; } - - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); - createSubscriptionIfNotExists(partition, topicName); - } catch (IOException e) { - log.error("Failed to create topic: [{}].", topicName.toString(), e); - throw new RuntimeException("Failed to create topic.", e); } - } - - @Override - public void destroy() { + topicAdminClient.createTopic(topicName); + topicSet.add(topicName.toString()); + log.info("Created new topic: [{}]", topicName.toString()); + createSubscriptionIfNotExists(partition, topicName); } private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { @@ -139,36 +135,27 @@ public class TbPubSubAdmin implements TbQueueAdmin { return; } - try (SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings)) { - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder() - .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) - .build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = - subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - - for (Subscription subscription : response.iterateAll()) { - if (subscription.getName().equals(subscriptionName.toString())) { - subscriptionSet.add(subscription.getName()); - return; - } + ListSubscriptionsRequest listSubscriptionsRequest = + ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); + SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); + for (Subscription subscription : response.iterateAll()) { + if (subscription.getName().equals(subscriptionName.toString())) { + subscriptionSet.add(subscription.getName()); + return; } + } - Subscription.Builder subscriptionBuilder = Subscription - .newBuilder() - .setName(subscriptionName.toString()) - .setTopic(topicName.toString()); + Subscription.Builder subscriptionBuilder = Subscription + .newBuilder() + .setName(subscriptionName.toString()) + .setTopic(topicName.toString()); - setAckDeadline(subscriptionBuilder); - setMessageRetention(subscriptionBuilder); + setAckDeadline(subscriptionBuilder); + setMessageRetention(subscriptionBuilder); - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); - } catch (IOException e) { - log.error("Failed to create subscription: [{}].", subscriptionName.toString(), e); - throw new RuntimeException("Failed to create subscription.", e); - } + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + subscriptionSet.add(subscriptionName.toString()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); } private void setAckDeadline(Subscription.Builder builder) { @@ -186,4 +173,14 @@ public class TbPubSubAdmin implements TbQueueAdmin { builder.setMessageRetentionDuration(duration); } } + + @Override + public void destroy() { + if (topicAdminClient != null) { + topicAdminClient.close(); + } + if (subscriptionAdminClient != null) { + subscriptionAdminClient.close(); + } + } } From 4fb309c37ea5854b1c7650c73b48db2bd86d5d56 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Apr 2020 13:30:11 +0300 Subject: [PATCH 087/196] moved kafka from service.js to own module --- .../service/script/RemoteJsInvokeService.java | 10 +- .../src/main/resources/thingsboard.yml | 6 +- .../api/jsInvokeMessageProcessor.js | 23 ++-- .../config/custom-environment-variables.yml | 2 + msa/js-executor/config/default.yml | 2 + msa/js-executor/queue/kafka/kafkaTemplate.js | 117 ++++++++++++++++++ msa/js-executor/server.js | 94 ++------------ 7 files changed, 152 insertions(+), 102 deletions(-) create mode 100644 msa/js-executor/queue/kafka/kafkaTemplate.js diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 8d1f9d662a..baf54bfe2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -46,17 +46,17 @@ import java.util.concurrent.atomic.AtomicInteger; @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Value("${js.remote.max_requests_timeout}") + @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; @Getter - @Value("${js.remote.max_errors}") +// @Value("${queue.js.max_errors}") private int maxErrors; - @Value("${js.remote.max_black_list_duration_sec:60}") + @Value("${queue.js.max_black_list_duration_sec:60}") private int maxBlackListDurationSec; - @Value("${js.remote.stats.enabled:false}") + @Value("${queue.js.stats.enabled:false}") private boolean statsEnabled; private final AtomicInteger kafkaPushedMsgs = new AtomicInteger(0); @@ -65,7 +65,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final AtomicInteger kafkaFailedMsgs = new AtomicInteger(0); private final AtomicInteger kafkaTimeoutMsgs = new AtomicInteger(0); - @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") +// @Scheduled(fixedDelayString = "${queue.js.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { int pushedMsgs = kafkaPushedMsgs.getAndSet(0); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e1296275cd..89ac618d6d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -415,7 +415,7 @@ state: persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" js: - evaluator: "${JS_EVALUATOR:local}" # local/remote + evaluator: "${JS_EVALUATOR:remote}" # local/remote # Built-in JVM JavaScript environment properties local: # Use Sandboxed (secured) JVM JavaScript environment @@ -582,9 +582,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index f0facf8cc1..4cf20b4227 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -31,6 +31,7 @@ const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); function JsInvokeMessageProcessor(producer) { + console.log("Kafka Producer:", producer); this.producer = producer; this.executor = new JsExecutor(useSandbox); this.scriptMap = {}; @@ -144,17 +145,17 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) { var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); - this.producer.send( - { - topic: responseTopic, - messages: [ - { - key: scriptId, - value: rawResponse, - headers: headers - } - ] - } + this.producer.send(responseTopic, scriptId, rawResponse, headers + // { + // topic: responseTopic, + // messages: [ + // { + // key: scriptId, + // value: rawResponse, + // headers: headers + // } + // ] + // } ).then( () => {}, (err) => { diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 585dfe8adb..77e68b0429 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,6 +14,8 @@ # limitations under the License. # +service-type: "TB_SERVICE_TYPE" + kafka: request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" bootstrap: diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 1290a8a429..724901d171 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,6 +14,8 @@ # limitations under the License. # +service-type: "kafka" + kafka: request_topic: "js.eval.requests" bootstrap: diff --git a/msa/js-executor/queue/kafka/kafkaTemplate.js b/msa/js-executor/queue/kafka/kafkaTemplate.js new file mode 100644 index 0000000000..bb3f81c48d --- /dev/null +++ b/msa/js-executor/queue/kafka/kafkaTemplate.js @@ -0,0 +1,117 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const { logLevel, Kafka } = require('kafkajs'); + +const config = require('config'), + JsInvokeMessageProcessor = require('../../api/jsInvokeMessageProcessor'), + logger = require('../../config/logger')._logger('main'), + KafkaJsWinstonLogCreator = require('../../config/logger').KafkaJsWinstonLogCreator; + +var kafkaClient; +var consumer; +var producer; + +function KafkaProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + return producer.send( + { + topic: responseTopic, + messages: [ + { + key: scriptId, + value: rawResponse, + headers: headers + } + ] + }); + } +} + +(async() => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + + const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); + const kafkaRequestTopic = config.get('kafka.request_topic'); + + logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); + logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); + + kafkaClient = new Kafka({ + brokers: kafkaBootstrapServers.split(','), + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator + }); + + consumer = kafkaClient.consumer({ groupId: 'js-executor-group' }); + producer = kafkaClient.producer(); + const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); + await consumer.connect(); + await producer.connect(); + await consumer.subscribe({ topic: kafkaRequestTopic}); + + logger.info('Started ThingsBoard JavaScript Executor Microservice.'); + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + messageProcessor.onJsInvokeMessage(message); + }, + }); + + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + if (consumer) { + logger.info('Stopping Kafka Consumer...'); + var _consumer = consumer; + consumer = null; + try { + await _consumer.disconnect(); + logger.info('Kafka Consumer stopped.'); + await disconnectProducer(); + process.exit(status); + } catch (e) { + logger.info('Kafka Consumer stop error.'); + await disconnectProducer(); + process.exit(status); + } + } else { + process.exit(status); + } +} + +async function disconnectProducer() { + if (producer) { + logger.info('Stopping Kafka Producer...'); + var _producer = producer; + producer = null; + try { + await _producer.disconnect(); + logger.info('Kafka Producer stopped.'); + } catch (e) { + logger.info('Kafka Producer stop error.'); + } + } +} diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index f56e5bb766..486f9dc187 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -13,89 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const { logLevel, Kafka } = require('kafkajs'); -const config = require('config'), - JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'), - logger = require('./config/logger')._logger('main'), - KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator; - -var kafkaClient; -var consumer; -var producer; - -(async() => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - - const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); - const kafkaRequestTopic = config.get('kafka.request_topic'); - - logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); - logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); - - kafkaClient = new Kafka({ - brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator - }); - - consumer = kafkaClient.consumer({ groupId: 'js-executor-group' }); - producer = kafkaClient.producer(); - const messageProcessor = new JsInvokeMessageProcessor(producer); - await consumer.connect(); - await producer.connect(); - await consumer.subscribe({ topic: kafkaRequestTopic}); - - logger.info('Started ThingsBoard JavaScript Executor Microservice.'); - await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - messageProcessor.onJsInvokeMessage(message); - }, - }); - - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -process.on('exit', () => { - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - if (consumer) { - logger.info('Stopping Kafka Consumer...'); - var _consumer = consumer; - consumer = null; - try { - await _consumer.disconnect(); - logger.info('Kafka Consumer stopped.'); - await disconnectProducer(); - process.exit(status); - } catch (e) { - logger.info('Kafka Consumer stop error.'); - await disconnectProducer(); - process.exit(status); - } - } else { - process.exit(status); - } +const config = require('config'); + +const serviceType = config.get('service-type'); +switch (serviceType) { + case 'kafka': + require('./queue/kafka/kafkaTemplate'); + console.log('Used kafka template.'); + break; + default: + console.error('Unknown service type: ', serviceType); + process.exit(-1); } -async function disconnectProducer() { - if (producer) { - logger.info('Stopping Kafka Producer...'); - var _producer = producer; - producer = null; - try { - await _producer.disconnect(); - logger.info('Kafka Producer stopped.'); - } catch (e) { - logger.info('Kafka Producer stop error.'); - } - } -} From c3bd2a0ebe2e7c005cae7520cc1ab6bf4e7224f8 Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Sat, 18 Apr 2020 00:33:24 +0300 Subject: [PATCH 088/196] Fixed incorrect params in the get timeseries request --- .../thingsboard/rest/client/RestClient.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 7e0dbf3858..f6a6541de0 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1646,7 +1646,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { addPageLinkToParam(params, pageLink); Map> timeseries = restTemplate.exchange( - baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParams(pageLink), + baseURL + "/api/plugins/telemetry/{entityType}/{entityId}/values/timeseries?keys={keys}&interval={interval}&agg={agg}&useStrictDataTypes={useStrictDataTypes}&" + getUrlParamsTs(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>>() { @@ -1998,17 +1998,25 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getUrlParams(TimePageLink pageLink) { - String urlParams = "limit={limit}&ascOrder={ascOrder}"; + return getUrlParams(pageLink, "startTime", "endTime"); + } + + private String getUrlParamsTs(TimePageLink pageLink) { + return getUrlParams(pageLink, "startTs", "endTs"); + } + + private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { + StringBuilder urlParams = new StringBuilder("limit={limit}&ascOrder={ascOrder}"); if (pageLink.getStartTime() != null) { - urlParams += "&startTime={startTime}"; + urlParams.append("&").append(startTime).append("={startTime}"); } if (pageLink.getEndTime() != null) { - urlParams += "&endTime={endTime}"; + urlParams.append("&").append(endTime).append("={endTime}"); } if (pageLink.getIdOffset() != null) { - urlParams += "&offset={offset}"; + urlParams.append("&offset={offset}"); } - return urlParams; + return urlParams.toString(); } private String getUrlParams(TextPageLink pageLink) { From eab6ba459824a9916ab1438418fcc6447c119bcd Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Mon, 20 Apr 2020 19:38:59 +0300 Subject: [PATCH 089/196] StringBuilder replaced with String --- .../java/org/thingsboard/rest/client/RestClient.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index f6a6541de0..652de133b7 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -2006,17 +2006,17 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } private String getUrlParams(TimePageLink pageLink, String startTime, String endTime) { - StringBuilder urlParams = new StringBuilder("limit={limit}&ascOrder={ascOrder}"); + String urlParams = "limit={limit}&ascOrder={ascOrder}"; if (pageLink.getStartTime() != null) { - urlParams.append("&").append(startTime).append("={startTime}"); + urlParams += "&" + startTime + "={startTime}"; } if (pageLink.getEndTime() != null) { - urlParams.append("&").append(endTime).append("={endTime}"); + urlParams += "&" + endTime + "={endTime}"; } if (pageLink.getIdOffset() != null) { - urlParams.append("&offset={offset}"); + urlParams += "&offset={offset}"; } - return urlParams.toString(); + return urlParams; } private String getUrlParams(TextPageLink pageLink) { From 283ad27cb54d476e011b92e4de94aac465a89c39 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Apr 2020 17:08:51 +0300 Subject: [PATCH 090/196] added rabbitmq queue arguments --- .../src/main/resources/thingsboard.yml | 6 ++ .../provider/AwsSqsTransportQueueFactory.java | 11 ++- .../InMemoryTbTransportQueueFactory.java | 9 +- .../RabbitMqMonolithQueueFactory.java | 61 +++++++++--- .../provider/RabbitMqTbCoreQueueFactory.java | 57 ++++++++--- .../RabbitMqTbRuleEngineQueueFactory.java | 48 +++++++-- .../RabbitMqTransportQueueFactory.java | 50 ++++++++-- .../ServiceBusTransportQueueFactory.java | 6 +- .../queue/rabbitmq/TbRabbitMqAdmin.java | 11 +-- .../rabbitmq/TbRabbitMqQueueArguments.java | 98 +++++++++++++++++++ .../src/main/resources/tb-coap-transport.yml | 6 ++ .../src/main/resources/tb-http-transport.yml | 6 ++ .../src/main/resources/tb-mqtt-transport.yml | 6 ++ 13 files changed, 319 insertions(+), 56 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0cae3252ff..0b2c40acef 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -566,6 +566,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index c9ba3ec15f..351cd4058c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -30,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -47,8 +48,10 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin coreAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; @@ -56,12 +59,15 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, TbAwsSqsQueueAttributes sqsQueueAttributes) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); } @@ -94,7 +100,7 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic()); + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); } @Override @@ -105,6 +111,9 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @PreDestroy private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } if (transportApiAdmin != null) { transportApiAdmin.destroy(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index c03f5e52d8..2855e577ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -32,9 +32,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @@ -43,13 +43,16 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; public InMemoryTbTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; } @Override @@ -86,7 +89,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic()); + return new InMemoryTbQueueProducer<>(coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index ff4a69e2e6..72f45e5276 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -28,8 +28,10 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -37,6 +39,8 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -48,7 +52,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -56,7 +65,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -64,73 +73,97 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 5708e0738c..d84bbedbd2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -34,13 +34,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -51,7 +55,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, TbQueueCoreSettings coreSettings, @@ -59,67 +68,91 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.rabbitMqSettings = rabbitMqSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), + return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index e2755938d9..a18f427fab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -32,13 +32,17 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -48,55 +52,63 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbRabbitMqSettings rabbitMqSettings, - TbQueueAdmin admin) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, coreSettings.getTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, ruleEngineSettings.getTopic(), + return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @@ -105,4 +117,20 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index dc6154255c..841e004b3f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -30,12 +30,17 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -43,34 +48,45 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbQueueCoreSettings coreSettings, + TbRabbitMqQueueArguments queueArguments) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; - this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + + this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); + this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); + this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbRabbitMqProducerTemplate> producerTemplate = - new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); TbRabbitMqConsumerTemplate> consumerTemplate = - new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,17 +97,33 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, transportApiSettings.getRequestsTopic()); + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), + return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index d38a4389a3..33c5277ca5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -29,6 +29,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -41,17 +42,20 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory private final TbServiceBusSettings serviceBusSettings; private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, TbQueueAdmin admin) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; } @Override @@ -82,7 +86,7 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 9a7e5db342..3bef6deb84 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -18,24 +18,23 @@ package org.thingsboard.server.queue.rabbitmq; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.Map; import java.util.concurrent.TimeoutException; @Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") public class TbRabbitMqAdmin implements TbQueueAdmin { private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; + private final Map arguments; - public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings) { + public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { this.rabbitMqSettings = rabbitMqSettings; + this.arguments = arguments; try { connection = rabbitMqSettings.getConnectionFactory().newConnection(); @@ -55,7 +54,7 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { @Override public void createTopicIfNotExists(String topic) { try { - channel.queueDeclare(topic, false, false, false, null); + channel.queueDeclare(topic, false, false, false, arguments); } catch (IOException e) { log.error("Failed to bind queue: [{}]", topic, e); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java new file mode 100644 index 0000000000..eb73e7dce6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.rabbitmq; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") +public class TbRabbitMqQueueArguments { + @Value("${queue.rabbitmq.queue-properties.core}") + private String coreProperties; + @Value("${queue.rabbitmq.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.rabbitmq.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.rabbitmq.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.rabbitmq.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreArgs; + @Getter + private Map ruleEngineArgs; + @Getter + private Map transportApiArgs; + @Getter + private Map notificationsArgs; + @Getter + private Map jsExecutorArgs; + + @PostConstruct + private void init() { + coreArgs = getArgs(coreProperties); + ruleEngineArgs = getArgs(ruleEngineProperties); + transportApiArgs = getArgs(transportApiProperties); + notificationsArgs = getArgs(notificationsProperties); + jsExecutorArgs = getArgs(jsExecutorProperties); + } + + private Map getArgs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String strValue = property.substring(delimiterPosition + 1); + configs.put(key, getObjectValue(strValue)); + } + return configs; + } + + private Object getObjectValue(String str) { + if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { + return Boolean.valueOf(str); + } else if (isNumeric(str)) { + return getNumericValue(str); + } + return str; + } + + private Object getNumericValue(String str) { + if (str.contains(".")) { + return Double.valueOf(str); + } else { + return Long.valueOf(str); + } + } + + private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); + + public boolean isNumeric(String strNum) { + if (strNum == null) { + return false; + } + return PATTERN.matcher(strNum).matches(); + } +} diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3e646c77c..c3dcb76508 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -95,6 +95,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0d48bda2b9..baac6e977f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -96,6 +96,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 4249757db1..e920891d66 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -126,6 +126,12 @@ queue: automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 53eb09bcb939ae28cbefefbbab5cb2dfe3d05434 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 22:31:25 +0300 Subject: [PATCH 091/196] Improvements to tests due to new architecture --- .../RuleChainActorMessageProcessor.java | 5 ++- .../server/actors/tenant/TenantActor.java | 40 ++++++++++++------- .../controller/TelemetryController.java | 8 ++-- .../queue/DefaultTbClusterService.java | 6 +++ .../DefaultTbRuleEngineConsumerService.java | 2 +- ...TbRuleEngineProcessingStrategyFactory.java | 16 ++++++-- .../rpc/DefaultTbCoreDeviceRpcService.java | 10 ++--- .../src/main/resources/thingsboard.yml | 2 +- ...tractMqttServerSideRpcIntegrationTest.java | 15 ++++--- .../AbstractMqttTelemetryIntegrationTest.java | 4 +- ...AbstractRuleEngineFlowIntegrationTest.java | 4 +- application/src/test/resources/logback.xml | 2 +- .../server/queue/memory/InMemoryStorage.java | 4 +- .../queue/memory/InMemoryTbQueueConsumer.java | 2 +- .../TbRuleEngineQueueConfiguration.java | 2 +- dao/src/test/resources/sql-test.properties | 10 ++++- 16 files changed, 87 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index a374e590b5..4e3d69c2b7 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -249,7 +249,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); + if (tenant == null) { + cantFindTenant = true; + log.info("[{}] Started tenant actor for missing tenant.", tenantId); + } else { + // This Service may be started for specific tenant only. + Optional isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); + isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); - if (isRuleEngineForCurrentTenant) { - try { - if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { - initRuleChains(); - } else { - isRuleEngineForCurrentTenant = false; + if (isRuleEngineForCurrentTenant) { + try { + if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenant.isIsolatedTbRuleEngine())) { + log.info("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + isRuleEngineForCurrentTenant = false; + } + } catch (Exception e) { + cantFindTenant = true; } - } catch (Exception e) { - cantFindTenant = true; } + log.info("[{}] Tenant actor started.", tenantId); } - log.info("[{}] Tenant actor started.", tenantId); } catch (Exception e) { log.warn("[{}] Unknown failure", tenantId, e); } @@ -104,7 +111,12 @@ public class TenantActor extends RuleChainManagerActor { @Override protected boolean process(TbActorMsg msg) { if (cantFindTenant) { - log.info("Missing Tenant msg: {}", msg); + log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); + if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { + QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; + queueMsg.getTbMsg().getCallback().onSuccess(); + } + return true; } switch (msg.getMsgType()) { case PARTITION_CHANGE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 8352e3f457..aee56652c4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -344,7 +344,7 @@ public class TelemetryController extends BaseController { return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdStr, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); @@ -354,13 +354,13 @@ public class TelemetryController extends BaseController { if (DataConstants.SERVER_SCOPE.equals(scope) || DataConstants.SHARED_SCOPE.equals(scope) || DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdStr, (result, tenantId, entityId) -> { + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { ListenableFuture> future = attributesService.removeAll(user.getTenantId(), entityId, scope, keys); Futures.addCallback(future, new FutureCallback>() { @Override public void onSuccess(@Nullable List tmp) { logAttributesDeleted(user, entityId, scope, keys, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); Set keysToNotify = new HashSet<>(); keys.forEach(key -> keysToNotify.add(new AttributeKey(scope, key))); @@ -397,7 +397,7 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityId.getEntityType() == EntityType.DEVICE) { + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { DeviceId deviceId = new DeviceId(entityId.getId()); tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( user.getTenantId(), deviceId, scope, attributes), null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f0054771d9..926b08f38b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -87,6 +87,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, msg.getTenantId(), msg.getDeviceId()); + log.trace("PUSHING msg: {} to:{}", msg, tpi); byte[] msgBytes = encodingService.encode(msg); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorNotificationMsg(ByteString.copyFrom(msgBytes)).build(); producerProvider.getTbCoreMsgProducer().send(tpi, new TbProtoQueueMsg<>(msg.getDeviceId().getId(), toCoreMsg), callback); @@ -96,6 +97,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits()) @@ -108,6 +110,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getRuleEngineMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); toRuleEngineMsgs.incrementAndGet(); } @@ -123,6 +126,7 @@ public class DefaultTbClusterService implements TbClusterService { } } TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -134,6 +138,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder() .setRequestIdMSB(response.getId().getMostSignificantBits()) .setRequestIdLSB(response.getId().getLeastSignificantBits()) @@ -147,6 +152,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId); + log.trace("PUSHING msg: {} to:{}", response, tpi); producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback); toTransportNfs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index c8f126b26c..6736518bcc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -174,7 +174,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< })); boolean timeout = false; - if (!ctx.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS)) { timeout = true; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index b6579b8dcb..bbf283e962 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.queue.processing; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration; @@ -32,7 +34,7 @@ public class TbRuleEngineProcessingStrategyFactory { public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) { switch (configuration.getType()) { - case "SKIP_ALL": + case "SKIP_ALL_FAILURES": return new SkipStrategy(name); case "RETRY_ALL": return new RetryStrategy(name, true, true, true, configuration); @@ -98,7 +100,7 @@ public class TbRuleEngineProcessingStrategyFactory { } log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { - toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, msg.getValue())); + toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); } if (pauseBetweenRetries > 0) { try { @@ -123,7 +125,15 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + if (!result.isSuccess()) { + log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + } + if (log.isTraceEnabled()) { + result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } + if (log.isTraceEnabled()) { + result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); + } return new TbRuleEngineProcessingDecision(true, null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java index 8a1423f97c..f6e5eb8b43 100644 --- a/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbCoreDeviceRpcService.java @@ -106,7 +106,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { @Override public void processRpcResponseFromRuleEngine(FromDeviceRpcResponse response) { - log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId()); + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), response); UUID requestId = response.getId(); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { @@ -177,9 +177,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private void scheduleToRuleEngineTimeout(ToDeviceRpcRequest request, UUID requestId) { long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to rule engine request: [{}]", this.hashCode(), requestId); + log.trace("[{}] processing to rule engine request.", requestId); scheduler.schedule(() -> { - log.trace("[{}] timeout for to rule engine request: [{}]", this.hashCode(), requestId); + log.trace("[{}] timeout for processing to rule engine request.", requestId); Consumer consumer = localToRuleEngineRpcRequests.remove(requestId); if (consumer != null) { consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT)); @@ -189,9 +189,9 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService { private void scheduleToDeviceTimeout(ToDeviceRpcRequest request, UUID requestId) { long timeout = Math.max(0, request.getExpirationTime() - System.currentTimeMillis()); - log.trace("[{}] processing to device request: [{}]", this.hashCode(), requestId); + log.trace("[{}] processing to device request.", requestId); scheduler.schedule(() -> { - log.trace("[{}] timeout for to device request: [{}]", this.hashCode(), requestId); + log.trace("[{}] timeout for to device request.", requestId); localToDeviceRpcRequests.remove(requestId); }, timeout, TimeUnit.MILLISECONDS); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0b2c40acef..3df3cb9559 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -622,7 +622,7 @@ queue: # For BATCH only batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch processing-strategy: - type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; diff --git a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java index 485d618a01..9ddb2c81d9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.service.security.AccessValidator; import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -55,6 +56,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC private User tenantAdmin; private Long asyncContextTimeoutToUseRpcPlugin; + private static final AtomicInteger atomicInteger = new AtomicInteger(2); + @Before public void beforeTest() throws Exception { @@ -70,7 +73,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setEmail("tenant" + atomicInteger.getAndIncrement() + "@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); @@ -130,7 +133,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"24\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -139,7 +142,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttOneWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"25\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/oneway/" + nonExistentDeviceId, setGpioRequest, String.class, @@ -169,7 +172,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC Thread.sleep(2000); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); @@ -187,7 +190,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"timeout\": 6000}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"27\",\"value\": 1},\"timeout\": 6000}"; String deviceId = savedDevice.getId().getId().toString(); doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isRequestTimeout(), @@ -196,7 +199,7 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractC @Test public void testServerMqttTwoWayRpcDeviceDoesNotExist() throws Exception { - String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; + String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"28\",\"value\": 1}}"; String nonExistentDeviceId = UUIDs.timeBased().toString(); String result = doPostAsync("/api/plugins/rpc/twoway/" + nonExistentDeviceId, setGpioRequest, String.class, diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 7e329d8b05..f96f207834 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -107,13 +107,13 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr CountDownLatch latch = new CountDownLatch(1); TestMqttCallback callback = new TestMqttCallback(client, latch); client.setCallback(callback); - client.connect(options).waitForCompletion(3000); + client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); String payload = "{\"key\":\"value\"}"; // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient <- SUB_ACK <- Transport - Thread.sleep(1000); + Thread.sleep(5000); doPostAsync("/api/plugins/telemetry/" + savedDevice.getId() + "/SHARED_SCOPE", payload, String.class, status().isOk()); latch.await(10, TimeUnit.SECONDS); assertEquals(payload, callback.getPayload()); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 3343d271fe..87a181deae 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -152,7 +152,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); // Pushing Message to the system actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); @@ -265,7 +265,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule // Pushing Message to the system actorSystem.tell(qMsg, ActorRef.noSender()); - Mockito.verify(tbMsgCallback, Mockito.timeout(3000)).onSuccess(); + Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); TimePageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 47dacce343..7943b9d9b1 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index e119d1433b..f51f5b8ae0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -50,10 +50,10 @@ public final class InMemoryStorage { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); } - public List get(String topic, long durationInMillis) throws InterruptedException { + public List get(String topic) throws InterruptedException { if (storage.containsKey(topic)) { List entities; - T first = (T) storage.get(topic).poll(durationInMillis, TimeUnit.MILLISECONDS); + T first = (T) storage.get(topic).poll(); if (first != null) { entities = new ArrayList<>(); entities.add(first); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 3920f143f1..a619eda132 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -68,7 +68,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon .stream() .map(tpi -> { try { - return storage.get(tpi.getFullTopicName(), durationInMillis); + return storage.get(tpi.getFullTopicName()); } catch (InterruptedException e) { if (!stopped) { log.error("Queue was interrupted.", e); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java index c5a24f20c6..019a76953b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbRuleEngineQueueConfiguration.java @@ -24,7 +24,7 @@ public class TbRuleEngineQueueConfiguration { private String topic; private int pollInterval; private int partitions; - private String packProcessingTimeout; + private long packProcessingTimeout; private TbRuleEngineQueueSubmitStrategyConfiguration submitStrategy; private TbRuleEngineQueueAckStrategyConfiguration processingStrategy; diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 13c0fbc818..7b5f82da2a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -35,4 +35,12 @@ service.type=monolith #spring.datasource.password=postgres #spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest #spring.datasource.driverClassName=org.postgresql.Driver -#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file +#spring.datasource.hikari.maximumPoolSize = 50 + +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST \ No newline at end of file From b2bb39f7a7c65c7e1ce8ee90ad17dcc6857ccf6e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 22:47:06 +0300 Subject: [PATCH 092/196] Restored WARN log level for tests --- application/src/test/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/resources/logback.xml b/application/src/test/resources/logback.xml index 7943b9d9b1..47dacce343 100644 --- a/application/src/test/resources/logback.xml +++ b/application/src/test/resources/logback.xml @@ -7,7 +7,7 @@ - + From 076cf655f87a914d1b882d35647faedaa8bd7c2e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 21 Apr 2020 23:54:26 +0300 Subject: [PATCH 093/196] Improved tests --- .../thingsboard/server/controller/TelemetryController.java | 5 ----- .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 2 +- dao/src/test/resources/sql-test.properties | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index aee56652c4..bb866e3e9b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -397,11 +397,6 @@ public class TelemetryController extends BaseController { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); - if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { - DeviceId deviceId = new DeviceId(entityId.getId()); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate( - user.getTenantId(), deviceId, scope, attributes), null); - } result.setResult(new ResponseEntity(HttpStatus.OK)); } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index f96f207834..0fe85fbb88 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -109,7 +109,7 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr client.setCallback(callback); client.connect(options).waitForCompletion(5000); client.subscribe("v1/devices/me/attributes", MqttQoS.AT_MOST_ONCE.value()); - String payload = "{\"key\":\"value\"}"; + String payload = "{\"key\":\"uniqueValue\"}"; // TODO 3.1: we need to acknowledge subscription only after it is processed by device actor and not when the message is pushed to queue. // MqttClient -> SUB REQUEST -> Transport -> Kafka -> Device Actor (subscribed) // MqttClient <- SUB_ACK <- Transport diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 7b5f82da2a..50d3a3feb3 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -37,6 +37,9 @@ service.type=monolith #spring.datasource.driverClassName=org.postgresql.Driver #spring.datasource.hikari.maximumPoolSize = 50 +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 + queue.rule-engine.queues[0].name=Main queue.rule-engine.queues[0].topic=tb_rule_engine.main queue.rule-engine.queues[0].poll-interval=25 From dc3d1dd794f1c70cbd6a6eecc9a14aaf7abe45e4 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 13:03:53 +0300 Subject: [PATCH 094/196] added ttl for events --- .../main/data/upgrade/2.4.3/schema_update_ttl.sql | 15 +++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 5 +++++ .../service/ttl/events/EventsCleanUpService.java | 5 ++--- .../server/dao/model/ModelConstants.java | 3 +++ .../server/dao/model/sql/EventEntity.java | 12 ++++++++++++ dao/src/main/resources/sql/schema-entities.sql | 15 +++++++++++++++ 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index ff1fb5129b..bd40434ffe 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,3 +123,18 @@ BEGIN END LOOP; END $$; + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ + DECLARE + ttl_ts bigint; +BEGIN + IF system_ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; + diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..d7454db0b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + try { + long ts = System.currentTimeMillis(); + conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //tiNOSONAR, ignoring because method used to execute thingsboard database upgrade script + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index bb7d112bb1..e10ea30893 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -50,8 +50,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - log.info("ttl: [{}]", ttl); - log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); - // TODO: 4/15/20 Do a clean up. + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..4a2f8aef23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,6 +32,9 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); + // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units + public static final long EPOCH_DIFF = 122192928000000000L; + /** * Generic constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..c0ad279d5b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -37,6 +37,9 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; @@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data @EqualsAndHashCode(callSuper = true) @@ -73,9 +77,13 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; From 0103da1ba4852f305606c7eb52c4ffd560f78dbc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 14:12:57 +0300 Subject: [PATCH 095/196] Improved Callback and Destroy of ndoes --- .../queue/DefaultTbCoreConsumerService.java | 15 +++- .../DefaultTbRuleEngineConsumerService.java | 8 +- .../service/queue/TbMsgPackCallback.java | 8 +- ...t.java => TbMsgPackProcessingContext.java} | 4 +- .../server/service/queue/TbPackCallback.java | 28 ++---- .../queue/TbPackProcessingContext.java | 90 +++++++++++++++++++ .../processing/AbstractConsumerService.java | 11 ++- .../TbRuleEngineProcessingResult.java | 6 +- ...va => TbMsgPackProcessingContextTest.java} | 4 +- 9 files changed, 124 insertions(+), 50 deletions(-) rename application/src/main/java/org/thingsboard/server/service/queue/{ProcessingAttemptContext.java => TbMsgPackProcessingContext.java} (96%) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java rename application/src/test/java/org/thingsboard/server/service/queue/{ProcessingAttemptContextTest.java => TbMsgPackProcessingContextTest.java} (94%) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c943e4c4d0..a5abd77ad5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -98,6 +99,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); pendingMap.forEach((id, msg) -> { log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + TbCallback callback = new TbPackCallback<>(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); if (toCoreMsg.hasToSubscriptionMgrMsg()) { @@ -147,8 +154,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } mainConsumer.commit(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 6736518bcc..0c48c54412 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -31,7 +31,6 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -64,9 +63,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; @Service @TbRuleEngineComponent @@ -116,10 +112,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @PreDestroy public void stop() { + super.destroy(); if (submitExecutor != null) { submitExecutor.shutdownNow(); } - ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config)); } @@ -156,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< submitStrategy.init(msgs); while (!stopped) { - ProcessingAttemptContext ctx = new ProcessingAttemptContext(submitStrategy); + TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(submitStrategy); submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> { log.trace("[{}] Creating callback for message: {}", id, msg.getValue()); ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java index 2a6b6a658d..f093cc885a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java @@ -18,21 +18,17 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeException; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; @Slf4j public class TbMsgPackCallback implements TbMsgCallback { private final UUID id; private final TenantId tenantId; - private final ProcessingAttemptContext ctx; + private final TbMsgPackProcessingContext ctx; - public TbMsgPackCallback(UUID id, TenantId tenantId, ProcessingAttemptContext ctx) { + public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) { this.id = id; this.tenantId = tenantId; this.ctx = ctx; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java rename to application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java index aefb1697cd..7e78ac6f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProcessingAttemptContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContext.java @@ -29,7 +29,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -public class ProcessingAttemptContext { +public class TbMsgPackProcessingContext { private final TbRuleEngineSubmitStrategy submitStrategy; @@ -44,7 +44,7 @@ public class ProcessingAttemptContext { @Getter private final ConcurrentMap exceptionsMap = new ConcurrentHashMap<>(); - public ProcessingAttemptContext(TbRuleEngineSubmitStrategy submitStrategy) { + public TbMsgPackProcessingContext(TbRuleEngineSubmitStrategy submitStrategy) { this.submitStrategy = submitStrategy; this.pendingMap = submitStrategy.getPendingMap(); this.pendingCount = new AtomicInteger(pendingMap.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index ba5a883ea0..5a5c172ee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -19,44 +19,26 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; @Slf4j public class TbPackCallback implements TbCallback { - private final CountDownLatch processingTimeoutLatch; - private final ConcurrentMap ackMap; - private final ConcurrentMap failedMap; + private final TbPackProcessingContext ctx; private final UUID id; - public TbPackCallback(UUID id, - CountDownLatch processingTimeoutLatch, - ConcurrentMap ackMap, - ConcurrentMap failedMap) { + public TbPackCallback(UUID id, TbPackProcessingContext ctx) { this.id = id; - this.processingTimeoutLatch = processingTimeoutLatch; - this.ackMap = ackMap; - this.failedMap = failedMap; + this.ctx = ctx; } @Override public void onSuccess() { log.trace("[{}] ON SUCCESS", id); - T msg = ackMap.remove(id); - if (msg != null && ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onSuccess(id); } @Override public void onFailure(Throwable t) { log.trace("[{}] ON FAILURE", id, t); - T msg = ackMap.remove(id); - if (msg != null) { - failedMap.put(id, msg); - } - if (ackMap.isEmpty()) { - processingTimeoutLatch.countDown(); - } + ctx.onFailure(id, t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java new file mode 100644 index 0000000000..e9f1224625 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackProcessingContext.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.extern.slf4j.Slf4j; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class TbPackProcessingContext { + + private final AtomicInteger pendingCount; + private final CountDownLatch processingTimeoutLatch; + private final ConcurrentMap ackMap; + private final ConcurrentMap failedMap; + + public TbPackProcessingContext(CountDownLatch processingTimeoutLatch, + ConcurrentMap ackMap, + ConcurrentMap failedMap) { + this.processingTimeoutLatch = processingTimeoutLatch; + this.pendingCount = new AtomicInteger(ackMap.size()); + this.ackMap = ackMap; + this.failedMap = failedMap; + } + + public boolean await(long packProcessingTimeout, TimeUnit milliseconds) throws InterruptedException { + return processingTimeoutLatch.await(packProcessingTimeout, milliseconds); + } + + public void onSuccess(UUID id) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + } + if (empty) { + processingTimeoutLatch.countDown(); + } else { + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T t : ackMap.values()) { + log.trace("left item: {}", t); + } + } + } + } + + public void onFailure(UUID id, Throwable t) { + boolean empty = false; + T msg = ackMap.remove(id); + if (msg != null) { + empty = pendingCount.decrementAndGet() == 0; + failedMap.put(id, msg); + if (log.isTraceEnabled()) { + log.trace("Items left: {}", ackMap.size()); + for (T v : ackMap.values()) { + log.trace("left item: {}", v); + } + } + } + if (empty) { + processingTimeoutLatch.countDown(); + } + } + + public ConcurrentMap getAckMap() { + return ackMap; + } + + public ConcurrentMap getFailedMap() { + return failedMap; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index a238e34337..c2705fcbdc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -23,11 +23,13 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.service.encoding.DataDecodingEncodingService; import org.thingsboard.server.service.queue.TbPackCallback; +import org.thingsboard.server.service.queue.TbPackProcessingContext; import javax.annotation.PreDestroy; import java.util.List; @@ -92,11 +94,12 @@ public abstract class AbstractConsumerService> pendingMap = msgs.stream().collect( Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); - ConcurrentMap> failedMap = new ConcurrentHashMap<>(); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); pendingMap.forEach((id, msg) -> { log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); - TbCallback callback = new TbPackCallback<>(id, processingTimeoutLatch, pendingMap, failedMap); + TbCallback callback = new TbPackCallback<>(id, ctx); try { handleNotification(id, msg, callback); } catch (Throwable e) { @@ -105,8 +108,8 @@ public abstract class AbstractConsumerService log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); - failedMap.forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue())); + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue())); } nfConsumer.commit(); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java index 8e0fcaa74a..e818ffb9d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.service.queue.ProcessingAttemptContext; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import java.util.UUID; import java.util.concurrent.ConcurrentMap; @@ -32,9 +32,9 @@ public class TbRuleEngineProcessingResult { @Getter private final boolean timeout; @Getter - private final ProcessingAttemptContext ctx; + private final TbMsgPackProcessingContext ctx; - public TbRuleEngineProcessingResult(boolean timeout, ProcessingAttemptContext ctx) { + public TbRuleEngineProcessingResult(boolean timeout, TbMsgPackProcessingContext ctx) { this.timeout = timeout; this.ctx = ctx; this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty(); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java similarity index 94% rename from application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java rename to application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java index 2de2414f71..43cd62ea97 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProcessingAttemptContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java @@ -37,7 +37,7 @@ import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) -public class ProcessingAttemptContextTest { +public class TbMsgPackProcessingContextTest { @Test public void testHighConcurrencyCase() throws InterruptedException { @@ -51,7 +51,7 @@ public class ProcessingAttemptContextTest { messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); } when(strategyMock.getPendingMap()).thenReturn(messages); - ProcessingAttemptContext context = new ProcessingAttemptContext(strategyMock); + TbMsgPackProcessingContext context = new TbMsgPackProcessingContext(strategyMock); for (UUID uuid : messages.keySet()) { for (int i = 0; i < parallelCount; i++) { executorService.submit(() -> context.onSuccess(uuid)); From 26a200dcfb1b610be6091b34edf7e76694ec8a3b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 14:27:16 +0300 Subject: [PATCH 096/196] Renaming the queues --- .../server/controller/QueueController.java | 2 +- .../TbRuleEngineSubmitStrategyFactory.java | 4 ++-- .../src/main/resources/thingsboard.yml | 19 +++++++++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java index 8d1cec7f4b..a0b46f9166 100644 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -43,7 +43,7 @@ public class QueueController extends BaseController { ServiceType type = ServiceType.valueOf(serviceType); switch (type) { case TB_RULE_ENGINE: - return Arrays.asList("HighPriority", "Main"); + return Arrays.asList("Main", "HighPriority", "SequentialByOriginator"); default: return Collections.emptyList(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java index f5a7457c17..90eb8561a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java @@ -29,9 +29,9 @@ public class TbRuleEngineSubmitStrategyFactory { return new BurstTbRuleEngineSubmitStrategy(name); case "BATCH": return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize()); - case "SEQUENTIAL_WITHIN_ORIGINATOR": + case "SEQUENTIAL_BY_ORIGINATOR": return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name); - case "SEQUENTIAL_WITHIN_TENANT": + case "SEQUENTIAL_BY_TENANT": return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name); case "SEQUENTIAL": return new SequentialTbRuleEngineSubmitStrategy(name); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3df3cb9559..9949a84c4d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -618,7 +618,7 @@ queue: partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: - type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch processing-strategy: @@ -633,7 +633,7 @@ queue: partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: - type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_WITHIN_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_WITHIN_ORIGINATOR, SEQUENTIAL_WITHIN_TENANT, SEQUENTIAL + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch processing-strategy: @@ -642,6 +642,21 @@ queue: retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:3}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" + submit-strategy: + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" From 71bf15caf79e7cd25f36227e969bf7910b3e0fc0 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:16:58 +0300 Subject: [PATCH 097/196] improvements --- .../data/upgrade/2.4.3/schema_update_ttl.sql | 24 ++++++++++++----- .../ttl/events/EventsCleanUpService.java | 7 +++-- .../src/main/resources/thingsboard.yml | 9 ++++--- .../resources/sql/schema-entities-hsql.sql | 27 +++++++++++++++++++ .../main/resources/sql/schema-entities.sql | 21 +++++++++++---- 5 files changed, 70 insertions(+), 18 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index bd40434ffe..dda3bd7b1c 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -124,17 +124,27 @@ BEGIN END $$; -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ - DECLARE - ttl_ts bigint; +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; BEGIN - IF system_ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; END $$; - diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index e10ea30893..a608ca257b 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -31,9 +31,12 @@ import java.sql.SQLException; @Service public class EventsCleanUpService extends AbstractCleanUpService { - @Value("${sql.ttl.events.events_key_value_ttl}") + @Value("${sql.ttl.events.events_ttl}") private long ttl; + @Value("${sql.ttl.events.debug_events_ttl}") + private long debugTtl; + @Value("${sql.ttl.events.enabled}") private boolean ttlTaskExecutionEnabled; @@ -50,7 +53,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6eee7bc32a..4b584ab44f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,12 +205,13 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day - events_key_value_ttl: "${SQL_TTL_EVENTS_EVENTS_KEY_VALUE_TTL:0}" # Number of seconds + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:900}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:900}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: @@ -371,7 +372,7 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ttl_test}" username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index f28f7f5ebd..d4900020c6 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -251,3 +252,29 @@ CREATE TABLE IF NOT EXISTS entity_view ( search_text varchar(255), additional_info varchar ); + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a70eb13ab7..931856e1df 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS event ( event_type varchar(255), event_uid varchar(255), tenant_id varchar(31), - ts bigint, + ts bigint NOT NULL, CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) ); @@ -253,16 +253,27 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ DECLARE ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; BEGIN - IF system_ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; END IF; + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; END $$; From a0b4a46c3da516c1405932128ac87e3f770b7797 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:20:23 +0300 Subject: [PATCH 098/196] fix typo --- application/src/main/resources/thingsboard.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4b584ab44f..cb343e8bdb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -205,13 +205,13 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:10000}" # Number of miliseconds. The current value corresponds to one day - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:900}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:900}" # Number of seconds. The current value corresponds to one week + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week # Actor system parameters actors: From 005d70d2eaf4051a28168c651d809574c9c98f6c Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:22:10 +0300 Subject: [PATCH 099/196] fix yml typo --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cb343e8bdb..456761a0cd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -372,7 +372,7 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ttl_test}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" username: "${SPRING_DATASOURCE_USERNAME:postgres}" password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: From ce086f915ea272504ce756c8e1c018a772c7a28b Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 16:53:38 +0300 Subject: [PATCH 100/196] try to pass travis --- .../thingsboard/server/install/ThingsboardInstallService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 842550d209..8dd45bc41b 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,8 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; From 6d20ed05501f4b2f42a5922f6f79a5f7b2af95d3 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 22 Apr 2020 18:09:47 +0300 Subject: [PATCH 101/196] fix event insert repository --- .../event/AbstractEventInsertRepository.java | 3 ++- .../sql/event/HsqlEventInsertRepository.java | 4 +-- .../sql/event/PsqlEventInsertRepository.java | 2 +- .../resources/sql/schema-entities-hsql.sql | 25 ------------------- .../main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-psql.sql | 2 +- 6 files changed, 7 insertions(+), 31 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java index b8d3af0968..8a341e0f81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/AbstractEventInsertRepository.java @@ -75,7 +75,8 @@ public abstract class AbstractEventInsertRepository implements EventInsertReposi .setParameter("entity_type", entity.getEntityType().name()) .setParameter("event_type", entity.getEventType()) .setParameter("event_uid", entity.getEventUid()) - .setParameter("tenant_id", entity.getTenantId()); + .setParameter("tenant_id", entity.getTenantId()) + .setParameter("ts", entity.getTs()); } private EventEntity processSaveOrUpdate(EventEntity entity, String query) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index bebfd7c5e0..f5a00fe7f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -44,7 +44,7 @@ public class HsqlEventInsertRepository extends AbstractEventInsertRepository { } private static String getInsertString(String conflictStatement) { - return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id" + - " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id)"; + return "MERGE INTO event USING (VALUES :id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) I (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) ON " + conflictStatement + " WHEN MATCHED THEN UPDATE SET event.id = I.id, event.body = I.body, event.entity_id = I.entity_id, event.entity_type = I.entity_type, event.event_type = I.event_type, event.event_uid = I.event_uid, event.tenant_id = I.tenant_id, event.ts = I.ts" + + " WHEN NOT MATCHED THEN INSERT (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (I.id, I.body, I.entity_id, I.entity_type, I.event_type, I.event_uid, I.tenant_id, I.ts)"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java index 1a8c12cb45..937262241a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java @@ -48,6 +48,6 @@ public class PsqlEventInsertRepository extends AbstractEventInsertRepository { } private static String getInsertOrUpdateString(String eventKeyStatement, String updateKeyStatement) { - return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, " + updateKeyStatement + " returning *"; + return "INSERT INTO event (id, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) VALUES (:id, :body, :entity_id, :entity_type, :event_type, :event_uid, :tenant_id, :ts) ON CONFLICT " + eventKeyStatement + " DO UPDATE SET body = :body, ts = :ts," + updateKeyStatement + " returning *"; } } \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index d4900020c6..697e34930a 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -253,28 +253,3 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN ttl bigint, IN debug_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - debug_ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type != %L::varchar AND event_type != %L::varchar) RETURNING *) SELECT count(*) FROM deleted', ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_ttl > 0 THEN - debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint AND (event_type = %L::varchar OR event_type = %L::varchar) RETURNING *) SELECT count(*) FROM deleted', debug_ttl_ts, 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; - diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 32e2a78620..bb0a964c13 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 3789444106..f83e80931b 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) ); -INSERT INTO tb_schema_settings (schema_version) VALUES (2005000); +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000; CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS From 5dbf7d764cceabb160fc168b579d4819c09e5b71 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Apr 2020 18:52:34 +0300 Subject: [PATCH 102/196] Default Dashboards --- .../data/json/demo/dashboards/gateways.json | 1274 +++++++++++++++++ .../dashboards/rule_engine_statistics.json | 520 +++++++ .../service/queue/TbCoreConsumerStats.java | 2 +- .../DefaultTelemetryWebSocketService.java | 8 +- .../src/main/resources/thingsboard.yml | 15 +- .../TbSynchronizationBeginNode.java | 2 +- .../transaction/TbSynchronizationEndNode.java | 2 +- 7 files changed, 1811 insertions(+), 12 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/gateways.json create mode 100644 application/src/main/data/json/demo/dashboards/rule_engine_statistics.json diff --git a/application/src/main/data/json/demo/dashboards/gateways.json b/application/src/main/data/json/demo/dashboards/gateways.json new file mode 100644 index 0000000000..f6370696c1 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/gateways.json @@ -0,0 +1,1274 @@ +{ + "title": "Gateways", + "configuration": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "isSystemType": true, + "bundleAlias": "entity_admin_widgets", + "typeAlias": "device_admin_table2", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "entitiesTitle": "List of gateways", + "enableSelectColumnDisplay": true, + "displayEntityLabel": false, + "entityNameColumnTitle": "Gateway Name" + }, + "title": "Devices gateway table", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value == 'false') {\n color = '#EB5757';\n} else {\n color = '#27AE60';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.3646047595211721 + }, + { + "name": "eventsSent", + "type": "timeseries", + "label": "Sent", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.7235710720767985 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Events", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5085933386303254 + }, + { + "name": "LOGS", + "type": "timeseries", + "label": "Latest log", + "color": "#ffc107", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3504240371585048, + "postFuncBody": "if(value) {\n return value.substring(0, 31) + \"...\";\n} else {\n return '';\n}" + }, + { + "name": "RemoteLoggingLevel", + "type": "attribute", + "label": "Log level", + "color": "#607d8b", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9785994222542516 + } + ], + "entityAliasId": "3e0f533a-0db1-3292-184f-06e73535061a" + } + ], + "showTitleIcon": true, + "titleIcon": "list", + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "List device", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "70837a9d-c3de-a9a7-03c5-dccd14998758", + "name": "Add device", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n\n", + "customCss": "", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddDeviceDialog();\n\nfunction openAddDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope', '$mdDialog',\n AddDeviceDialogController\n ],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddDeviceDialogController($scope, $mdDialog) {\n let vm = this;\n vm.types = types;\n vm.attributes = {};\n vm.deviceType = \"gateway\";\n\n vm.cancel = () => {\n $mdDialog.hide();\n };\n\n vm.save = () => {\n vm.loading = true;\n $scope.addDeviceForm.$setPristine();\n let device = {\n additionalInfo: {gateway: true},\n name: vm.deviceName,\n type: vm.deviceType,\n label: vm.deviceLabel\n };\n deviceService.saveDevice(device).then(\n (device) => {\n saveAttributes(device.id).then(\n () => {\n vm.loading = false;\n updateAliasData();\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n };\n\n function saveAttributes(entityId) {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({\n key: key,\n value: vm.attributes[key]\n });\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(\n entityId.entityType, entityId.id,\n \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n\n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController\n .resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController\n .setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController\n .getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast(\n 'widgetForceReInit');\n });\n }\n}" + } + ], + "actionCellButton": [ + { + "id": "78845501-234e-a452-6819-82b5b776e99f", + "name": "Configuration", + "icon": "settings", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "openRightLayout": false, + "setEntityId": true + }, + { + "id": "f6ffdba8-e40f-2b8d-851b-f5ecaf18606b", + "name": "Graphs", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname_grafic", + "setEntityId": true + }, + { + "id": "242671f3-76c6-6982-7acc-6f12addf0ccc", + "name": "Edit device", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit device

\n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n \n
\n
Device name is required.
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n
\n \n \n \n \n
\n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": "/*=======================================================================*/\n/*========== There are two examples: for edit and add entity ==========*/\n/*=======================================================================*/\n/*======================== Edit entity example ========================*/\n/*=======================================================================*/\n/*\n.edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n\n.relations-list.old-relations tb-entity-select tb-entity-autocomplete button {\n display: none;\n} \n*/\n/*========================================================================*/\n/*========================= Add entity example =========================*/\n/*========================================================================*/\n/*\n.add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n.relations-list .header {\n padding-right: 5px;\n padding-bottom: 5px;\n padding-left: 5px;\n}\n\n.relations-list .header .cell {\n padding-right: 5px;\n padding-left: 5px;\n font-size: 12px;\n font-weight: 700;\n color: rgba(0, 0, 0, .54);\n white-space: nowrap;\n}\n\n.relations-list .body {\n padding-right: 5px;\n padding-bottom: 15px;\n padding-left: 5px;\n}\n\n.relations-list .body .row {\n padding-top: 5px;\n}\n\n.relations-list .body .cell {\n padding-right: 5px;\n padding-left: 5px;\n}\n\n.relations-list .body md-autocomplete-wrap md-input-container {\n height: 30px;\n}\n\n.relations-list .body .md-button {\n margin: 0;\n}\n*/\n", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n \nopenEditDeviceDialog();\n\nfunction openEditDeviceDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditDeviceDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditDeviceDialogController($scope,$mdDialog) {\n let vm = this;\n vm.types = types;\n vm.loading = false;\n vm.attributes = {};\n \n getEntityInfo();\n \n function getEntityInfo() {\n vm.loading = true;\n deviceService.getDevice(entityId.id).then(\n (device) => {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n (data) => {\n if (data.length) {\n getEntityAttributes(data);\n }\n vm.device = device;\n vm.loading = false;\n } \n );\n }\n )\n }\n \n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n vm.save = () => {\n vm.loading = true;\n $scope.editDeviceForm.$setPristine();\n deviceService.saveDevice(vm.device).then(\n () => {\n saveAttributes().then(\n () => {\n updateAliasData();\n vm.loading = false;\n $mdDialog.hide();\n }\n );\n },\n () => {\n vm.loading = false;\n }\n );\n }\n \n function getEntityAttributes(attributes) {\n for (let i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n }\n \n function saveAttributes() {\n let attributesArray = [];\n for (let key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when([]);\n }\n }\n \n function updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n console.log(widgetContext);\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + }, + { + "id": "862ec2b7-fbcf-376e-f85f-b77c07f36efa", + "name": "Delete device", + "icon": "delete", + "type": "custom", + "customFunction": "let $injector = widgetContext.$scope.$injector;\nlet $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n types = $injector.get('types'),\n deviceService = $injector.get('deviceService'),\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteDeviceDialog();\n\nfunction openDeleteDeviceDialog() {\n let title = \"Are you sure you want to delete the device \" + entityName + \"?\";\n let content = \"Be careful, after the confirmation, the device and all related data will become unrecoverable!\";\n let confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(() => {\n deleteDevice();\n })\n}\n\nfunction deleteDevice() {\n deviceService.deleteDevice(entityId.id).then(\n () => {\n updateAliasData();\n }\n );\n}\n\nfunction updateAliasData() {\n let aliasIds = [];\n for (let id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n let tasks = [];\n aliasIds.forEach((aliasId) => {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(() => {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}" + } + ], + "rowClick": [ + { + "id": "ad5fc7e1-5e60-e056-6940-a75a383466a1", + "name": "to_entityname__config", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "__entityname__config", + "setEntityId": true, + "stateEntityParamName": "" + } + ] + } + }, + "id": "94715984-ae74-76e4-20b7-2f956b01ed80" + }, + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 14, + "sizeY": 13, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "LOGS", + "type": "timeseries", + "label": "LOGS", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.3496649158709739, + "postFuncBody": "return value.replace(/ - (.*) - \\[/gi, ' - $1 - [');" + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 2592000000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Debug events (logs)", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "eadabbc7-519e-76fc-ba10-b3fe8c18da10" + }, + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "opcuaEventsProduced", + "type": "timeseries", + "label": "opcuaEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.1477920581839779 + }, + { + "name": "opcuaEventsSent", + "type": "timeseries", + "label": "opcuaEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6500957113784758 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "f928afc4-30d1-8d0c-e3cf-777f9f9d1155" + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 7, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "Events", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.8156044798125357 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "Produced", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.6538259344015449 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 604800000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 6, + "hideEmptyLines": true + }, + "title": "Total Messages", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": false, + "showMax": false, + "showAvg": true, + "showTotal": false + } + }, + "id": "2a95b473-042d-59d0-2da2-40d0cccb6c8a" + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "eventsSent", + "type": "timeseries", + "label": "eventsSent", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.41414001784591314 + }, + { + "name": "eventsProduced", + "type": "timeseries", + "label": "eventsProduced", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.7819101846284422 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "History information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "aaa69366-aacc-9028-65aa-645c0f8533ec" + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 17, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "bleEventsProduced", + "type": "timeseries", + "label": "bleEventsProduced", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.5625165504526104 + }, + { + "name": "bleEventsSent", + "type": "timeseries", + "label": "bleEventsSent", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "tooltipValueFormatter": "", + "showSeparateAxis": false, + "axisTitle": "", + "axisPosition": "left", + "axisTicksFormatter": "", + "comparisonSettings": { + "showValuesForComparison": true, + "comparisonValuesLabel": "", + "color": "" + } + }, + "_hash": 0.6817950080745288 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "interval": 5000, + "timewindowMs": 120000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Real time information", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "right", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": true + }, + "actions": {} + }, + "id": "ce5c7d01-a3ef-5cf0-4578-8505135c23a0" + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "attributes_card", + "type": "latest", + "title": "New widget", + "sizeX": 7, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "eventsTitle": "Gateway Events Form", + "eventsReg": [ + "EventsProduced", + "EventsSent" + ] + }, + "title": "Gateway events", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "466f046d-6005-a168-b107-60fcb2469cd5" + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "isSystemType": true, + "bundleAlias": "gateway_widgets", + "typeAlias": "config_form_latest", + "type": "latest", + "title": "New widget", + "sizeX": 10, + "sizeY": 9, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "gatewayTitle": "Gateway configuration (Single device)", + "readOnly": false + }, + "title": "New Gateway configuration (Single device)", + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "dropShadow": true, + "enableFullscreen": true, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "displayTimewindow": true, + "showLegend": false, + "actions": {} + }, + "id": "8fc32225-164f-3258-73f7-e6b6d959cf0b" + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodParams": "{}", + "methodName": "gateway_reboot", + "buttonText": "GATEWAY REBOOT" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "063fc179-c9fd-f952-e714-f24e9c43c05c" + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "isSystemType": true, + "bundleAlias": "control_widgets", + "typeAlias": "rpcbutton", + "type": "rpc", + "title": "New widget", + "sizeX": 4, + "sizeY": 2, + "config": { + "targetDeviceAliases": [], + "showTitle": false, + "backgroundColor": "#e6e7e8", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "0px", + "settings": { + "requestTimeout": 5000, + "oneWayElseTwoWay": true, + "styleButton": { + "isRaised": true, + "isPrimary": false + }, + "methodName": "gateway_restart", + "methodParams": "{}", + "buttonText": "gateway restart" + }, + "title": "New RPC Button", + "dropShadow": true, + "enableFullscreen": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "datasources": [], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true, + "targetDeviceAliasIds": [ + "b2487e75-2fa4-f211-142c-434dfd50c70c" + ] + }, + "id": "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7" + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 4, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.9743324774725604 + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.5530093635101525 + } + ], + "entityAliasId": "b2487e75-2fa4-f211-142c-434dfd50c70c" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Latitude: ${latitude:7}
Longitude: ${longitude:7}

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [], + "useMarkerImageFunction": false, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.Mapnik", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 5 + }, + "title": "Gateway Location", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "6770b6ba-eff8-df05-75f8-c1f9326d4842" + } + }, + "states": { + "main_gateway": { + "name": "Gateways", + "root": true, + "layouts": { + "main": { + "widgets": { + "94715984-ae74-76e4-20b7-2f956b01ed80": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname__config": { + "name": "${entityName} Configuration", + "root": false, + "layouts": { + "main": { + "widgets": { + "eadabbc7-519e-76fc-ba10-b3fe8c18da10": { + "sizeX": 14, + "sizeY": 13, + "row": 0, + "col": 10 + }, + "8fc32225-164f-3258-73f7-e6b6d959cf0b": { + "sizeX": 10, + "sizeY": 9, + "row": 0, + "col": 0 + }, + "063fc179-c9fd-f952-e714-f24e9c43c05c": { + "sizeX": 4, + "sizeY": 2, + "row": 9, + "col": 0 + }, + "3c2134cc-27a0-93e1-dbe1-2fa7c1ce16b7": { + "sizeX": 4, + "sizeY": 2, + "row": 11, + "col": 0 + }, + "6770b6ba-eff8-df05-75f8-c1f9326d4842": { + "sizeX": 6, + "sizeY": 4, + "row": 9, + "col": 4 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "__entityname_grafic": { + "name": "${entityName} Details", + "root": false, + "layouts": { + "main": { + "widgets": { + "f928afc4-30d1-8d0c-e3cf-777f9f9d1155": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 4, + "col": 7 + }, + "2a95b473-042d-59d0-2da2-40d0cccb6c8a": { + "sizeX": 7, + "sizeY": 7, + "row": 5, + "col": 0 + }, + "aaa69366-aacc-9028-65aa-645c0f8533ec": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 0, + "col": 7 + }, + "ce5c7d01-a3ef-5cf0-4578-8505135c23a0": { + "sizeX": 17, + "sizeY": 4, + "mobileHeight": null, + "row": 8, + "col": 7 + }, + "466f046d-6005-a168-b107-60fcb2469cd5": { + "sizeX": 7, + "sizeY": 5, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "auto 100%", + "autoFillHeight": true, + "mobileAutoFillHeight": true, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "3e0f533a-0db1-3292-184f-06e73535061a": { + "id": "3e0f533a-0db1-3292-184f-06e73535061a", + "alias": "Gateways", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "gateway", + "deviceNameFilter": "" + } + }, + "b2487e75-2fa4-f211-142c-434dfd50c70c": { + "id": "b2487e75-2fa4-f211-142c-434dfd50c70c", + "alias": "Current Gateway", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 25000 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "settings": { + "stateControllerId": "entity", + "showTitle": true, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true, + "titleColor": "rgba(0,0,0,0.870588)" + } + }, + "name": "Gateways" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json new file mode 100644 index 0000000000..b8231ab880 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -0,0 +1,520 @@ +{ + "title": "Rule Engine Statistics", + "configuration": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "successfulMsgs", + "type": "timeseries", + "label": "${entityName} Successful", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.15490750967648736 + }, + { + "name": "failedMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Failures", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.4186621166514697 + }, + { + "name": "tmpFailed", + "type": "timeseries", + "label": "${entityName} Processing Failures", + "color": "#ffc107", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.49891007198715376 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Queue Stats", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "81987f19-3eac-e4ce-b790-d96e9b54d9a0" + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "timeseries_table", + "type": "timeseries", + "title": "New widget", + "sizeX": 24, + "sizeY": 5, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Chain", + "color": "#2196f3", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleChainName;" + }, + "_hash": 0.9954481282345906 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Rule Node", + "color": "#4caf50", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).ruleNodeName;" + }, + "_hash": 0.18580357036589978 + }, + { + "name": "ruleEngineException", + "type": "timeseries", + "label": "Latest Error", + "color": "#f44336", + "settings": { + "useCellStyleFunction": false, + "useCellContentFunction": true, + "cellContentFunction": "return JSON.parse(value).message;" + }, + "_hash": 0.7255162989552142 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showTimestamp": true, + "displayPagination": true, + "defaultPageSize": 10 + }, + "title": "Exceptions", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": false, + "showLegend": false, + "widgetStyle": {}, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "5eb79712-5c24-3060-7e4f-6af36b8f842d" + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 12, + "sizeY": 7, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "timeoutMsgs", + "type": "timeseries", + "label": "${entityName} Permanent Timeouts", + "color": "#4caf50", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.565222981550328 + }, + { + "name": "tmpTimeout", + "type": "timeseries", + "label": "${entityName} Processing Timeouts", + "color": "#9c27b0", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": false, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.2679547062508352 + } + ], + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + } + ], + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 300000 + }, + "aggregation": { + "type": "NONE", + "limit": 8640 + }, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + } + }, + "title": "Processing Failures and Timeouts", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "actions": {}, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": false, + "showTotal": true + } + }, + "id": "ad3f1417-87a8-750e-fc67-49a2de1466d4" + } + }, + "states": { + "default": { + "name": "Rule Engine Statistics", + "root": true, + "layouts": { + "main": { + "widgets": { + "81987f19-3eac-e4ce-b790-d96e9b54d9a0": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 0 + }, + "5eb79712-5c24-3060-7e4f-6af36b8f842d": { + "sizeX": 24, + "sizeY": 5, + "row": 7, + "col": 0 + }, + "ad3f1417-87a8-750e-fc67-49a2de1466d4": { + "sizeX": 12, + "sizeY": 7, + "mobileHeight": null, + "row": 0, + "col": 12 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "140f23dd-e3a0-ed98-6189-03c49d2d8018": { + "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", + "alias": "TbServiceQueues", + "filter": { + "type": "assetType", + "resolveMultiple": true, + "assetType": "TbServiceQueue", + "assetNameFilter": "" + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1586176634823, + "endTimeMs": 1586263034823 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Rule Engine Statistics" +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 78319997a9..c362c5e889 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -79,7 +79,7 @@ public class TbCoreConsumerStats { public void printStats() { int total = totalCounter.getAndSet(0); if (total > 0) { - log.info("Transport total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + + log.info("Total [{}] sessionEvents [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] subInfo [{}] claimDevice [{}]" + " deviceState [{}] subMgr [{}] coreNfs [{}]", total, sessionEventCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index 5e9e83b8aa..6afcaf97bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -345,6 +345,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi keys.forEach(key -> subState.put(key, 0L)); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -353,7 +355,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(false) .keyStates(subState) - .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + .scope(scope).build(); subService.addSubscription(sub); } @@ -440,6 +442,8 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); + TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.SERVER_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -448,7 +452,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(true) .keyStates(subState) - .scope(TbAttributeSubscriptionScope.valueOf(cmd.getScope())).build(); + .scope(scope).build(); subService.addSubscription(sub); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9949a84c4d..6a16349c4c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -410,8 +410,9 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" state: - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}" + # Should be greater then transport.sessions.report_timeout + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" js: @@ -589,7 +590,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic @@ -615,7 +616,7 @@ queue: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -630,7 +631,7 @@ queue: - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:3}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -643,9 +644,9 @@ queue: failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" - topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.hp}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:3}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:5}" pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java index 31501688eb..751b061666 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationBeginNode.java @@ -33,7 +33,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization start", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Starts synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "This node should be used together with \"synchronization end\" node. \n This node will put messages into queue based on message originator id. \n" + "Subsequent messages will not be processed until the previous message processing is completed or timeout event occurs.\n" + "Size of the queue per originator and timeout values are configurable on a system level", diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java index ac3d42ef36..8b83bff1fa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transaction/TbSynchronizationEndNode.java @@ -35,7 +35,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; type = ComponentType.ACTION, name = "synchronization end", configClazz = EmptyNodeConfiguration.class, - nodeDescription = "Stops synchronization of message processing based on message originator", + nodeDescription = "This Node is now deprecated. Use \"Checkpoint\" instead.", nodeDetails = "", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = ("tbNodeEmptyConfig") From 087ea2d691b900d4f8cafc9dd055f7cad7019137 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Apr 2020 09:25:25 +0300 Subject: [PATCH 103/196] Commented Unstable Test --- .../mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java index 0fe85fbb88..47e9537ef9 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/telemetry/AbstractMqttTelemetryIntegrationTest.java @@ -97,7 +97,8 @@ public abstract class AbstractMqttTelemetryIntegrationTest extends AbstractContr assertEquals("4", values.get("key4").get(0).get("value")); } - @Test + +// @Test - Unstable public void testMqttQoSLevel() throws Exception { String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient(MQTT_URL, clientId); From 9a052ef0bc333c8864b6ac9bc228d6290e0629d3 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Thu, 23 Apr 2020 12:39:33 +0300 Subject: [PATCH 104/196] fix typo in EventEntity class --- .../java/org/thingsboard/server/dao/model/sql/EventEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index 0a541e24b0..3790577144 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -113,6 +113,6 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity Date: Thu, 23 Apr 2020 14:10:45 +0300 Subject: [PATCH 105/196] Changed default partition sizes --- application/src/main/resources/thingsboard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6a16349c4c..7457cc8c4c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -616,7 +616,7 @@ queue: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -631,7 +631,7 @@ queue: - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL @@ -646,7 +646,7 @@ queue: - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:5}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL From 1510fe8448002e5c0f73a7ed957a0d52721df8dc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 12:22:34 +0300 Subject: [PATCH 106/196] Default Dashboard and Rule Chain --- .../json/demo/dashboards/theromstats.json | 1236 +++++++++++++++++ .../demo/rule_chains/root_rule_chain.json | 146 ++ .../demo/rule_chains/thermostat_alarms.json | 137 ++ .../DefaultSystemDataLoaderService.java | 36 +- .../service/install/InstallScripts.java | 53 +- 5 files changed, 1604 insertions(+), 4 deletions(-) create mode 100644 application/src/main/data/json/demo/dashboards/theromstats.json create mode 100644 application/src/main/data/json/demo/rule_chains/root_rule_chain.json create mode 100644 application/src/main/data/json/demo/rule_chains/thermostat_alarms.json diff --git a/application/src/main/data/json/demo/dashboards/theromstats.json b/application/src/main/data/json/demo/dashboards/theromstats.json new file mode 100644 index 0000000000..bd191a6bc5 --- /dev/null +++ b/application/src/main/data/json/demo/dashboards/theromstats.json @@ -0,0 +1,1236 @@ +{ + "title": "Thermostats", + "configuration": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "isSystemType": true, + "bundleAlias": "cards", + "typeAlias": "entities_table", + "type": "latest", + "title": "New widget", + "sizeX": 11, + "sizeY": 11, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSearch": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "entityName", + "displayEntityName": true, + "displayEntityType": false, + "enableSelectColumnDisplay": false, + "entitiesTitle": "Thermostats", + "displayEntityLabel": false, + "entityNameColumnTitle": "Thermostat name" + }, + "title": "Thermostats", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "active", + "type": "attribute", + "label": "Active", + "color": "#2196f3", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": true, + "useCellContentFunction": true, + "cellContentFunction": "value = '⬤';\nreturn value;", + "cellStyleFunction": "var color;\nif (value === \"true\") {\n color = 'rgb(39, 134, 34)';\n} else {\n color = 'rgb(255, 0, 0)';\n}\nreturn {\n color: color,\n fontSize: '18px'\n};" + }, + "_hash": 0.9264526512320641 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#4caf50", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.9801965063904188, + "units": "°C", + "decimals": 1 + }, + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#f44336", + "settings": { + "columnWidth": "0px", + "useCellStyleFunction": false, + "useCellContentFunction": false + }, + "_hash": 0.5726727868178358, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": { + "headerButton": [ + { + "id": "85b803db-90f2-5c63-1388-a378e0eb10d6", + "name": "Edit location", + "icon": "map", + "type": "openDashboardState", + "targetDashboardStateId": "map", + "setEntityId": false + }, + { + "id": "8ab5a518-67d2-b6a2-956d-81fd512294b2", + "name": "Add", + "icon": "add", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Add thermostat

\n \n \n \n \n
\n \n \n
\n \n \n \n
\n
Thermostat name is required.
\n
\n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Create\n Cancel\n \n
\n", + "customCss": ".add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n saveEntityPromise().then(\n function (entity) {\n saveAttributes(entity.id);\n updateAliasData();\n $mdDialog.hide();\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + } + ], + "actionCellButton": [ + { + "id": "ca241cd8-788d-5508-a9ce-74b03ef42a7f", + "name": "Chart", + "icon": "show_chart", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + }, + { + "id": "7506576f-87ba-d3a0-88fb-e304d451776d", + "name": "Edit", + "icon": "edit", + "type": "customPretty", + "customHtml": "\n
\n \n
\n

Edit thermostat {{vm.entityName}}

\n \n \n \n \n
\n \n \n
\n \n \n \n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Save\n Cancel\n \n
\n", + "customCss": ".edit-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.edit-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.edit-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n attributeService = $injector.get('attributeService');\n\nopenEditEntityDialog();\n\nfunction openEditEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', EditEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction EditEntityDialogController($scope,$mdDialog) {\n var vm = this;\n vm.entityId = entityId;\n vm.entityName = entityName;\n vm.entityType = entityId.entityType;\n vm.attributes = {};\n vm.serverAttributes = {};\n getEntityInfo();\n \n vm.save = function() {\n saveAttributes();\n $mdDialog.hide();\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n function getEntityAttributes(attributes) {\n for (var i = 0; i < attributes.length; i++) {\n vm.attributes[attributes[i].key] = attributes[i].value; \n }\n vm.serverAttributes = angular.copy(vm.attributes);\n }\n \n function getEntityInfo() {\n attributeService.getEntityAttributesValues(entityId.entityType, entityId.id, 'SERVER_SCOPE').then(\n function(data){\n if (data.length) {\n getEntityAttributes(data);\n }\n });\n }\n \n function saveAttributes() {\n var attributesArray = [];\n for (var key in vm.attributes) {\n if (vm.attributes[key] !== vm.serverAttributes[key]) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n}" + }, + { + "id": "3488848b-e47d-6af6-659f-5d78369ece5e", + "name": "Delete", + "icon": "delete", + "type": "custom", + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n deviceService = $injector.get('deviceService')\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteEntityDialog();\n\nfunction openDeleteEntityDialog() {\n var title = 'Delete thermostat \"' + entityName + '\"';\n var content = 'Are you sure you want to delete the thermostat \"' +\n entityName + '\"?';\n var confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(function() {\n deleteEntity();\n })\n}\n\nfunction deleteEntity() {\n deviceService.deleteDevice(entityId.id).then(\n function success() {\n updateAliasData();\n },\n function fail() {\n showErrorDialog();\n }\n );\n}\n\nfunction updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}\n\nfunction showErrorDialog() {\n var title = 'Error';\n var content = 'An error occurred while deleting the thermostat. Please try again.';\n var alert = $mdDialog.alert()\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .parent(angular.element($document[0].body))\n .targetEvent($event)\n .multiple(true)\n .clickOutsideToClose(true)\n .ok('CLOSE');\n $mdDialog.show(alert);\n}" + } + ], + "rowClick": [ + { + "id": "e3928f23-c135-0766-71d5-65ed61e0ce8d", + "name": "show alarm", + "icon": "more_horiz", + "type": "updateDashboardState", + "targetDashboardStateId": "default", + "setEntityId": true, + "stateEntityParamName": "alarm" + } + ] + } + }, + "id": "f33c746c-0dfc-c212-395b-b448c8a17209" + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "isSystemType": true, + "bundleAlias": "alarm_widgets", + "typeAlias": "alarms_table", + "type": "alarm", + "title": "New widget", + "sizeX": 13, + "sizeY": 5, + "config": { + "timewindow": { + "realtime": { + "interval": 1000, + "timewindowMs": 86400000 + }, + "aggregation": { + "type": "NONE", + "limit": 200 + } + }, + "showTitle": true, + "backgroundColor": "rgb(255, 255, 255)", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "4px", + "settings": { + "enableSelection": true, + "enableSearch": true, + "displayDetails": true, + "allowAcknowledgment": true, + "allowClear": true, + "displayPagination": true, + "defaultPageSize": 10, + "defaultSortOrder": "-createdTime", + "enableSelectColumnDisplay": false, + "enableStatusFilter": true, + "alarmsTitle": "Alarms" + }, + "title": "New Alarms table", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400, + "padding": "5px 10px 5px 10px" + }, + "useDashboardTimewindow": false, + "showLegend": false, + "alarmSource": { + "type": "entity", + "dataKeys": [ + { + "name": "createdTime", + "type": "alarm", + "label": "Created time", + "color": "#2196f3", + "settings": {}, + "_hash": 0.7308410188824108 + }, + { + "name": "originator", + "type": "alarm", + "label": "Originator", + "color": "#4caf50", + "settings": {}, + "_hash": 0.056085530105439485 + }, + { + "name": "type", + "type": "alarm", + "label": "Type", + "color": "#f44336", + "settings": {}, + "_hash": 0.10212012352561795 + }, + { + "name": "severity", + "type": "alarm", + "label": "Severity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1777349980531262 + }, + { + "name": "status", + "type": "alarm", + "label": "Status", + "color": "#607d8b", + "settings": {}, + "_hash": 0.7977920750136249 + } + ], + "entityAliasId": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "name": "alarms" + }, + "alarmSearchStatus": "ANY", + "alarmsPollingInterval": 5, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "displayTimewindow": true, + "actions": {}, + "datasources": [], + "alarmsMaxCountLoad": 0, + "alarmsFetchSize": 100 + }, + "id": "7943196b-eedb-d422-f9c3-b32d379ad172" + }, + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "Temperature", + "color": "#ef5350", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.7852346160709658, + "units": "°C", + "decimals": 1 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Temperature", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "14a19183-f0b2-d6be-0f62-9863f0a51111" + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "isSystemType": true, + "bundleAlias": "charts", + "typeAlias": "basic_timeseries", + "type": "timeseries", + "title": "New widget", + "sizeX": 18, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "humidity", + "type": "timeseries", + "label": "Humidity", + "color": "#2196f3", + "settings": { + "excludeFromStacking": false, + "hideDataByDefault": false, + "disableDataHiding": false, + "removeFromLegend": false, + "showLines": true, + "fillLines": true, + "showPoints": false, + "showPointShape": "circle", + "pointShapeFormatter": "var size = radius * Math.sqrt(Math.PI) / 2;\nctx.moveTo(x - size, y - size);\nctx.lineTo(x + size, y + size);\nctx.moveTo(x - size, y + size);\nctx.lineTo(x + size, y - size);", + "showPointsLineWidth": 5, + "showPointsRadius": 3, + "showSeparateAxis": false, + "axisPosition": "left", + "thresholds": [ + { + "thresholdValueSource": "predefinedValue" + } + ], + "comparisonSettings": { + "showValuesForComparison": true + } + }, + "_hash": 0.28640715926957183, + "units": "%", + "decimals": 0 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "interval": 30000, + "timewindowMs": 3600000 + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "shadowSize": 4, + "fontColor": "#545454", + "fontSize": 10, + "xaxis": { + "showLabels": true, + "color": "#545454" + }, + "yaxis": { + "showLabels": true, + "color": "#545454" + }, + "grid": { + "color": "#545454", + "tickColor": "#DDDDDD", + "verticalLines": true, + "horizontalLines": true, + "outlineWidth": 1 + }, + "stack": false, + "tooltipIndividual": false, + "timeForComparison": "months", + "xaxisSecond": { + "axisPosition": "top", + "showLabels": true + }, + "smoothLines": true + }, + "title": "Humidity", + "dropShadow": true, + "enableFullscreen": true, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "mobileHeight": null, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "widgetStyle": {}, + "useDashboardTimewindow": false, + "displayTimewindow": true, + "showLegend": true, + "legendConfig": { + "direction": "column", + "position": "bottom", + "showMin": true, + "showMax": true, + "showAvg": true, + "showTotal": false + }, + "actions": {} + }, + "id": "07f49fd5-a73b-d74c-c220-362c20af81f4" + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "update_multiple_attributes", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "alarmTemperature", + "type": "attribute", + "label": "High temperature alarm", + "color": "#4caf50", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.8725278440159361 + }, + { + "name": "thresholdTemperature", + "type": "attribute", + "label": "High temperature threshold, °C", + "color": "#f44336", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmTemperature" + }, + "_hash": 0.7316078472857874 + }, + { + "name": "alarmHumidity", + "type": "attribute", + "label": "Low humidity alarm", + "color": "#ffc107", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "booleanCheckbox", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1 + }, + "_hash": 0.5339673667431057 + }, + { + "name": "thresholdHumidity", + "type": "attribute", + "label": "Low humidity threshold, %", + "color": "#607d8b", + "settings": { + "dataKeyType": "server", + "dataKeyValueType": "double", + "required": false, + "isEditable": "editable", + "dataKeyHidden": false, + "step": 1, + "disabledOnDataKey": "alarmHumidity" + }, + "_hash": 0.2687091190358901 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": true, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "showActionButtons": false, + "showResultMessage": true, + "fieldsAlignment": "column", + "fieldsInRow": 2, + "groupTitle": "${entityName}", + "widgetTitle": "Termostat settings" + }, + "title": "New Update Multiple Attributes", + "dropShadow": true, + "enableFullscreen": false, + "enableDataExport": false, + "widgetStyle": {}, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "actions": {}, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "c4631f94-2db3-523b-4d09-2a1a0a75d93f" + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "isSystemType": true, + "bundleAlias": "maps_v2", + "typeAlias": "openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 13, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#2196f3", + "settings": {}, + "_hash": 0.1371919646686739, + "decimals": 1, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#4caf50", + "settings": {}, + "_hash": 0.043177186765847475, + "decimals": 0, + "postFuncBody": "return value || \"\";" + }, + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#f44336", + "settings": {}, + "_hash": 0.5548964320315584 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#ffc107", + "settings": {}, + "_hash": 0.1803778014971602 + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.30926987994082844 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Thermostat details
", + "markerImageSize": 48, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "colorFunction": "\n", + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "defaultCenterPosition": [ + 0, + 0 + ], + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "useLabelFunction": true, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "defaultZoomLevel": 14, + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;" + }, + "title": "Thermostat maps", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "headerButton": [], + "tooltipAction": [ + { + "id": "bef25673-b37a-8821-bc0f-5d6dd3680f24", + "name": "navigate_to_details", + "icon": "more_horiz", + "type": "openDashboardState", + "targetDashboardStateId": "chart", + "setEntityId": true + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb" + }, + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 24, + "sizeY": 12, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "68a058e1-fdda-8482-715b-3ae4a488568e" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "00fb2742-ba1f-7e43-673f-d6c08b72ed06" + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "isSystemType": true, + "bundleAlias": "input_widgets", + "typeAlias": "markers_placement_openstreetmap", + "type": "latest", + "title": "New widget", + "sizeX": 6, + "sizeY": 6, + "config": { + "datasources": [ + { + "type": "entity", + "dataKeys": [ + { + "name": "longitude", + "type": "attribute", + "label": "longitude", + "color": "#2196f3", + "settings": {}, + "_hash": 0.3640193654284214 + }, + { + "name": "latitude", + "type": "attribute", + "label": "latitude", + "color": "#4caf50", + "settings": {}, + "_hash": 0.49020393887695923 + }, + { + "name": "temperature", + "type": "timeseries", + "label": "temperature", + "color": "#f44336", + "settings": {}, + "_hash": 0.5885892766009955, + "postFuncBody": "return value || \"\";" + }, + { + "name": "humidity", + "type": "timeseries", + "label": "humidity", + "color": "#ffc107", + "settings": {}, + "_hash": 0.21077893588180707, + "postFuncBody": "return value || \"\";" + }, + { + "name": "active", + "type": "attribute", + "label": "active", + "color": "#607d8b", + "settings": {}, + "_hash": 0.34722983638504346 + } + ], + "entityAliasId": "12ae98c7-1ea2-52cf-64d5-763e9d993547" + } + ], + "timewindow": { + "realtime": { + "timewindowMs": 60000 + } + }, + "showTitle": false, + "backgroundColor": "#fff", + "color": "rgba(0, 0, 0, 0.87)", + "padding": "8px", + "settings": { + "fitMapBounds": true, + "latKeyName": "latitude", + "lngKeyName": "longitude", + "showLabel": true, + "label": "${entityName}", + "tooltipPattern": "${entityName}

Temperature: ${temperature:1} °C
Humidity: ${humidity:0} %

Delete", + "markerImageSize": 34, + "useColorFunction": false, + "markerImages": [ + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiNmNDQzMzZ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+", + "data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJzdmc0NDA4IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHg9IjAiIHk9IjAiIHZpZXdCb3g9IjAgMCAxNTAgMTUwIiB4bWw6c3BhY2U9InByZXNlcnZlIj48c3R5bGU+LnN0MntmaWxsOiMyNzg2MjJ9PC9zdHlsZT48ZyBpZD0ibGF5ZXIxIj48ZyBpZD0icGF0aDY4ODEtMy01LTUtMS04LTQtNC03LTgiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0xNDYuNDM4IC0yNzYuMDI4KSIgb3BhY2l0eT0iLjg5MiI+PHJhZGlhbEdyYWRpZW50IGlkPSJTVkdJRF8xXyIgY3g9IjMwODUuMjE1IiBjeT0iMzE3OC40NTgiIHI9IjQ5LjkwMSIgZ3JhZGllbnRUcmFuc2Zvcm09Im1hdHJpeCguNjc5MyAuMDA3NiAtLjUwOSAuNTYxMiAtMjMyLjYyOSAtMTQxMS43MjUpIiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+PHN0b3Agb2Zmc2V0PSIwIi8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLW9wYWNpdHk9Ii4xODgiLz48L3JhZGlhbEdyYWRpZW50PjxwYXRoIGQ9Ik0yODUuNiAzODguNWMxMC4zLTEyLjQgNC40LTIyLjQtMTQuNC0yMi40LTE4LjkgMC00Mi40IDEwLTUzLjkgMjIuNC0xNi44IDE4IC40IDIzLjUtLjIgMzUtLjEgMS44IDMuOSAxLjggNyAwIDE5LjgtMTEuNSA0Ni41LTE3IDYxLjUtMzUiIGZpbGw9InVybCgjU1ZHSURfMV8pIi8+PC9nPjxwYXRoIGlkPSJwYXRoNjg4MS0zLTUtNS0xLTgtNC00IiBjbGFzcz0ic3QyIiBkPSJNMTI0LjcgNjkuMWMtLjktMjcuNS0yMi4zLTQ5LjgtNDkuOC00OS44cy00OSAyMi4zLTQ5LjggNDkuOGMtMS4zIDQwLjEgMzAuNyA1Mi4yIDQ0LjcgNzggMi4yIDQgOCA0IDEwLjEgMCAxNC4xLTI1LjggNDYuMS0zNy45IDQ0LjgtNzgiLz48L2c+PGcgaWQ9Imc0OTI4Ij48Y2lyY2xlIGlkPSJwYXRoNDk3OCIgY2xhc3M9InN0MiIgY3g9Ijc0LjkiIGN5PSI2OS4xIiByPSI0OS45Ii8+PGcgaWQ9Imc0OTE1Ij48cGF0aCBpZD0icGF0aDY4ODMtMi0zLTUtMi00LTktNC05IiBkPSJNNzQuOCAxMDYuNGMtMjAuNiAwLTM3LjQtMTYuNy0zNy40LTM3LjQgMC0yMC42IDE2LjctMzcuNCAzNy40LTM3LjQgMjAuNiAwIDM3LjQgMTYuNyAzNy40IDM3LjRzLTE2LjcgMzcuNC0zNy40IDM3LjQiIGZpbGw9IiNmZmYiLz48L2c+PC9nPjxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik05NS45IDQ2LjZWNDloLTEwdi0yLjVsMTAgLjF6bS0yIDUuM2gtOHYyLjVoOHYtMi41em0tOCA3LjloNnYtMi41aC02djIuNXptNCAyLjloLTR2Mi41aDR2LTIuNXptLTQgNy44aDJWNjhoLTJ2Mi41em0xLjUgMTRjMCA2LjktNS41IDEyLjUtMTIuMyAxMi41cy0xMi4zLTUuNi0xMi4zLTEyLjVjMC00LjUgMi4zLTguNSA2LjEtMTAuN1Y0NS41YzAtMy41IDIuOC02LjMgNi4yLTYuM3M2LjIgMi44IDYuMiA2LjN2MjguM2MzLjggMi4yIDYuMSA2LjMgNi4xIDEwLjd6bS0yLjQgMGMwLTMuOC0yLjEtNy4yLTUuNC04LjlsLS43LS4zVjQ1LjVjMC0yLjEtMS43LTMuOC0zLjgtMy44LTIuMSAwLTMuOCAxLjctMy44IDMuOHYyOS44bC0uNy4zYy0zLjMgMS43LTUuNCA1LjEtNS40IDguOSAwIDUuNSA0LjQgMTAgOS45IDEwUzg1IDkwIDg1IDg0LjV6bS0yLjEgMGMwIDQuNC0zLjUgOC03LjggOHMtNy44LTMuNi03LjgtOGMwLTMuNiAyLjQtNi44IDUuOC03LjdsLjUtLjFWNjEuNWgzLjF2MTUuMmwuNS4xYzMuMyAxIDUuNyA0LjEgNS43IDcuN3ptLTcuNC01LjNjLS4yLS44LTEtMS40LTEuOS0xLjItMyAuNy01IDMuMy01IDYuNCAwIC45LjcgMS42IDEuNiAxLjZzMS42LS43IDEuNi0xLjZjMC0xLjYgMS4xLTMgMi42LTMuMy43LS4yIDEuMy0xIDEuMS0xLjl6Ii8+PC9zdmc+Cg==" + ], + "useMarkerImageFunction": true, + "color": "#fe7569", + "mapProvider": "OpenStreetMap.HOT", + "showTooltip": true, + "autocloseTooltip": true, + "defaultCenterPosition": [ + 0, + 0 + ], + "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + "showTooltipAction": "click", + "polygonKeyName": "coordinates", + "polygonOpacity": 0.5, + "polygonStrokeOpacity": 1, + "polygonStrokeWeight": 1, + "zoomOnClick": true, + "showCoverageOnHover": true, + "animate": true, + "maxClusterRadius": 80, + "removeOutsideVisibleBounds": true, + "defaultZoomLevel": 12, + "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", + "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", + "useLabelFunction": true + }, + "title": "New Markers Placement - OpenStreetMap", + "dropShadow": true, + "enableFullscreen": false, + "titleStyle": { + "fontSize": "16px", + "fontWeight": 400 + }, + "useDashboardTimewindow": true, + "showLegend": false, + "widgetStyle": {}, + "actions": { + "tooltipAction": [ + { + "id": "54c293c4-9ca6-e34f-dc6a-0271944c1c66", + "name": "delete", + "icon": "more_horiz", + "type": "custom", + "customFunction": "var $rootScope = widgetContext.$scope.$injector.get('$rootScope');\nvar entityDatasource = widgetContext.map.subscription.datasources.filter(\n function(entity) {\n return entity.entityId === entityId.id\n });\n\nwidgetContext.map.saveMarkerLocation(entityDatasource[0],\n widgetContext.map.locations[0], {\n \"lat\": null,\n \"lng\": null\n }).then(function succes() {\n $rootScope.$broadcast('widgetForceReInit');\n });" + } + ] + }, + "showTitleIcon": false, + "titleIcon": null, + "iconColor": "rgba(0, 0, 0, 0.87)", + "iconSize": "24px", + "titleTooltip": "", + "displayTimewindow": true + }, + "id": "0a430429-9078-9ae6-2b67-e4a15a2bf8bf" + } + }, + "states": { + "default": { + "name": "Thermostat", + "root": true, + "layouts": { + "main": { + "widgets": { + "f33c746c-0dfc-c212-395b-b448c8a17209": { + "sizeX": 11, + "sizeY": 11, + "row": 0, + "col": 0 + }, + "7943196b-eedb-d422-f9c3-b32d379ad172": { + "sizeX": 13, + "sizeY": 5, + "row": 0, + "col": 11 + }, + "3da9a9a1-0b9a-2e1f-0dcb-0ff34a695abb": { + "sizeX": 13, + "sizeY": 6, + "row": 5, + "col": 11 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "map": { + "name": "Edit location", + "root": false, + "layouts": { + "main": { + "widgets": { + "00fb2742-ba1f-7e43-673f-d6c08b72ed06": { + "sizeX": 24, + "sizeY": 12, + "row": 0, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + }, + "chart": { + "name": "${entityName}", + "root": false, + "layouts": { + "main": { + "widgets": { + "14a19183-f0b2-d6be-0f62-9863f0a51111": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 0, + "col": 6 + }, + "07f49fd5-a73b-d74c-c220-362c20af81f4": { + "sizeX": 18, + "sizeY": 6, + "mobileHeight": null, + "row": 6, + "col": 6 + }, + "c4631f94-2db3-523b-4d09-2a1a0a75d93f": { + "sizeX": 6, + "sizeY": 6, + "row": 0, + "col": 0 + }, + "0a430429-9078-9ae6-2b67-e4a15a2bf8bf": { + "sizeX": 6, + "sizeY": 6, + "row": 6, + "col": 0 + } + }, + "gridSettings": { + "backgroundColor": "#eeeeee", + "color": "rgba(0,0,0,0.870588)", + "columns": 24, + "margins": [ + 10, + 10 + ], + "backgroundSizeMode": "100%", + "autoFillHeight": true, + "mobileAutoFillHeight": false, + "mobileRowHeight": 70 + } + } + } + } + }, + "entityAliases": { + "68a058e1-fdda-8482-715b-3ae4a488568e": { + "id": "68a058e1-fdda-8482-715b-3ae4a488568e", + "alias": "Thermostats", + "filter": { + "type": "deviceType", + "resolveMultiple": true, + "deviceType": "thermostat", + "deviceNameFilter": "" + } + }, + "12ae98c7-1ea2-52cf-64d5-763e9d993547": { + "id": "12ae98c7-1ea2-52cf-64d5-763e9d993547", + "alias": "Thermostat", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": null, + "defaultStateEntity": null + } + }, + "ce27a9d0-93bf-b7a4-054d-d0369a8cf813": { + "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", + "alias": "Thermostat-alarm", + "filter": { + "type": "stateEntity", + "resolveMultiple": false, + "stateEntityParamName": "alarm", + "defaultStateEntity": null + } + } + }, + "timewindow": { + "displayValue": "", + "selectedTab": 0, + "hideInterval": false, + "hideAggregation": false, + "hideAggInterval": false, + "realtime": { + "interval": 1000, + "timewindowMs": 60000 + }, + "history": { + "historyType": 0, + "interval": 1000, + "timewindowMs": 60000, + "fixedTimewindow": { + "startTimeMs": 1587473857304, + "endTimeMs": 1587560257304 + } + }, + "aggregation": { + "type": "AVG", + "limit": 25000 + } + }, + "settings": { + "stateControllerId": "entity", + "showTitle": false, + "showDashboardsSelect": true, + "showEntitiesSelect": true, + "showDashboardTimewindow": true, + "showDashboardExport": true, + "toolbarAlwaysOpen": true + } + }, + "name": "Thermostats" +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json new file mode 100644 index 0000000000..1b321b1e1e --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -0,0 +1,146 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Root Rule Chain", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 2, + "nodes": [ + { + "additionalInfo": { + "layoutX": 824, + "layoutY": 156 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", + "name": "Save Timeseries", + "debugMode": false, + "configuration": { + "defaultTTL": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 52 + }, + "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", + "name": "Save Client Attributes", + "debugMode": false, + "configuration": { + "scope": "CLIENT_SCOPE" + } + }, + { + "additionalInfo": { + "layoutX": 347, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", + "name": "Message Type Switch", + "debugMode": false, + "configuration": { + "version": 0 + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 266 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log RPC from Device", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 379 + }, + "type": "org.thingsboard.rule.engine.action.TbLogNode", + "name": "Log Other", + "debugMode": false, + "configuration": { + "jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" + } + }, + { + "additionalInfo": { + "layoutX": 825, + "layoutY": 468 + }, + "type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", + "name": "RPC Call Request", + "debugMode": false, + "configuration": { + "timeoutInSeconds": 60 + } + }, + { + "additionalInfo": { + "layoutX": 1069, + "layoutY": 90 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Is Thermostat?", + "debugMode": false, + "configuration": { + "jsScript": "return metadata[\"deviceType\"] === \"thermostat\";" + } + } + ], + "connections": [ + { + "fromIndex": 0, + "toIndex": 6, + "type": "Success" + }, + { + "fromIndex": 2, + "toIndex": 4, + "type": "Other" + }, + { + "fromIndex": 2, + "toIndex": 1, + "type": "Post attributes" + }, + { + "fromIndex": 2, + "toIndex": 0, + "type": "Post telemetry" + }, + { + "fromIndex": 2, + "toIndex": 3, + "type": "RPC Request from Device" + }, + { + "fromIndex": 2, + "toIndex": 5, + "type": "RPC Request to Device" + } + ], + "ruleChainConnections": [ + { + "fromIndex": 6, + "targetRuleChainId": { + "entityType": "RULE_CHAIN", + "id": "83d42540-85fd-11ea-aee2-794850541ced" + }, + "additionalInfo": { + "layoutX": 1088, + "layoutY": 203, + "ruleChainNodeId": "rule-chain-node-9" + }, + "type": "True" + } + ] + } +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json new file mode 100644 index 0000000000..9b7e3401d2 --- /dev/null +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -0,0 +1,137 @@ +{ + "ruleChain": { + "additionalInfo": null, + "name": "Thermostat Alarms", + "firstRuleNodeId": null, + "root": false, + "debugMode": false, + "configuration": null + }, + "metadata": { + "firstNodeIndex": 5, + "nodes": [ + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 67 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.temperature;\nreturn details;", + "severity": "MAJOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [] + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 201 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Temp Alarm", + "debugMode": false, + "configuration": { + "alarmType": "High Temperature", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 930, + "layoutY": 131 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateAlarmNode", + "name": "Create Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\ndetails.triggerValue = msg.humidity;\nreturn details;", + "severity": "MINOR", + "propagate": true, + "useMessageAlarmData": false, + "relationTypes": [] + } + }, + { + "additionalInfo": { + "layoutX": 929, + "layoutY": 275 + }, + "type": "org.thingsboard.rule.engine.action.TbClearAlarmNode", + "name": "Clear Humidity Alarm", + "debugMode": false, + "configuration": { + "alarmType": "Low Humidity", + "alarmDetailsBuildJs": "var details = {};\nif (metadata.prevAlarmDetails) {\n details = JSON.parse(metadata.prevAlarmDetails);\n}\nreturn details;" + } + }, + { + "additionalInfo": { + "layoutX": 586, + "layoutY": 148 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", + "name": "Check Alarms", + "debugMode": true, + "configuration": { + "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" + } + }, + { + "additionalInfo": { + "layoutX": 321, + "layoutY": 149 + }, + "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", + "name": "Fetch Configuration", + "debugMode": true, + "configuration": { + "clientAttributeNames": [], + "sharedAttributeNames": [], + "serverAttributeNames": [ + "alarmTemperature", + "thresholdTemperature", + "alarmHumidity", + "thresholdHumidity" + ], + "latestTsKeyNames": [], + "tellFailureIfAbsent": false, + "getLatestValueWithTs": false + } + } + ], + "connections": [ + { + "fromIndex": 4, + "toIndex": 0, + "type": "NewTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 1, + "type": "ClearTempAlarm" + }, + { + "fromIndex": 4, + "toIndex": 2, + "type": "NewHumidityAlarm" + }, + { + "fromIndex": 4, + "toIndex": 3, + "type": "ClearHumidityAlarm" + }, + { + "fromIndex": 5, + "toIndex": 4, + "type": "Success" + } + ], + "ruleChainConnections": null + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index fd907714fa..f024a328c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,24 +25,32 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.util.Arrays; + @Service @Profile("install") @Slf4j @@ -76,6 +84,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private DeviceService deviceService; + @Autowired + private AttributesService attributesService; + @Autowired private DeviceCredentialsService deviceCredentialsService; @@ -120,7 +131,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { demoTenant.setRegion("Global"); demoTenant.setTitle("Tenant"); demoTenant = tenantService.saveTenant(demoTenant); - installScripts.createDefaultRuleChains(demoTenant.getId()); + installScripts.loadDemoRuleChains(demoTenant.getId()); createUser(Authority.TENANT_ADMIN, demoTenant.getId(), null, "tenant@thingsboard.org", "tenant"); Customer customerA = new Customer(); @@ -152,6 +163,25 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + "Raspberry Pi GPIO control sample application"); + DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + + attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 20)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 50)))); + + attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE, + Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmTemperature", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("alarmHumidity", true)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdTemperature", (long) 25)), + new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("thresholdHumidity", (long) 30)))); + installScripts.loadDashboards(demoTenant.getId(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 73e1dbf7fd..d61ca861eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,6 +24,7 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -182,4 +183,54 @@ public class InstallScripts { } + public void loadDemoRuleChains(TenantId tenantId) throws Exception { + Path ruleChainsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, RULE_CHAINS_DIR); + try { + JsonNode ruleChainJson = objectMapper.readTree(ruleChainsDir.resolve("thermostat_alarms.json").toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + ruleChain.setTenantId(tenantId); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + + JsonNode rootChainJson = objectMapper.readTree(ruleChainsDir.resolve("root_rule_chain.json").toFile()); + RuleChain rootChain = objectMapper.treeToValue(rootChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData rootChainMetaData = objectMapper.treeToValue(rootChainJson.get("metadata"), RuleChainMetaData.class); + + RuleChainId thermostatsRuleChainId = ruleChain.getId(); + rootChainMetaData.getRuleChainConnections().forEach(connection -> connection.setTargetRuleChainId(thermostatsRuleChainId)); + rootChain.setTenantId(tenantId); + rootChain = ruleChainService.saveRuleChain(rootChain); + rootChainMetaData.setRuleChainId(rootChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), rootChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json", e); + throw new RuntimeException("Unable to load dashboard from json", e); + } + + try (DirectoryStream dirStream = Files.newDirectoryStream(ruleChainsDir, path -> path.toString().endsWith(JSON_EXT))) { + dirStream.forEach( + path -> { + try { + JsonNode ruleChainJson = objectMapper.readTree(path.toFile()); + RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); + RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); + + ruleChain.setTenantId(tenantId); + if (ruleChain.getName().equals("Root Rule Chain")) { + ruleChain.setRoot(true); + } + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + ruleChainMetaData.setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); + } catch (Exception e) { + log.error("Unable to load dashboard from json: [{}]", path.toString()); + throw new RuntimeException("Unable to load dashboard from json", e); + } + } + ); + } + } } From 9b0e92cac30159ea1bb1fc2eeb0fc978f4fa8d72 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 12:46:48 +0300 Subject: [PATCH 107/196] New Default Dashboard and Rule Chains --- .../server/service/install/DefaultSystemDataLoaderService.java | 2 +- .../org/thingsboard/server/service/install/InstallScripts.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index f024a328c7..b8d5d26d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index d61ca861eb..37f724ad22 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From a91a69c57f6737fbdd438e5aeb79eb5bf47e9a13 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 13:55:06 +0300 Subject: [PATCH 108/196] Fixed Docker Compose --- .../service/install/InstallScripts.java | 24 ------------- .../service/script/RemoteJsInvokeService.java | 2 +- docker/docker-compose.yml | 21 ++++------- docker/kafka.env | 2 +- docker/tb-coap-transport.env | 1 + docker/tb-http-transport.env | 1 + docker/tb-mqtt-transport.env | 1 + docker/tb-node.env | 4 +-- docker/tb-node/conf/logback.xml | 4 +-- docker/tb-node/conf/thingsboard.conf | 2 +- docker/tb-transports/coap/conf/logback.xml | 4 +-- .../coap/conf/tb-coap-transport.conf | 2 +- docker/tb-transports/http/conf/logback.xml | 4 +-- .../http/conf/tb-http-transport.conf | 2 +- docker/tb-transports/mqtt/conf/logback.xml | 4 +-- .../mqtt/conf/tb-mqtt-transport.conf | 2 +- k8s/tb-coap-transport-configmap.yml | 6 ++-- k8s/tb-http-transport-configmap.yml | 6 ++-- k8s/tb-mqtt-transport-configmap.yml | 6 ++-- k8s/tb-node-configmap.yml | 6 ++-- k8s/thingsboard.yml | 36 +++++++------------ .../src/main/resources/tb-coap-transport.yml | 2 +- .../src/main/resources/tb-http-transport.yml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 2 +- 24 files changed, 53 insertions(+), 93 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 37f724ad22..5181834b80 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -208,29 +208,5 @@ public class InstallScripts { log.error("Unable to load dashboard from json", e); throw new RuntimeException("Unable to load dashboard from json", e); } - - try (DirectoryStream dirStream = Files.newDirectoryStream(ruleChainsDir, path -> path.toString().endsWith(JSON_EXT))) { - dirStream.forEach( - path -> { - try { - JsonNode ruleChainJson = objectMapper.readTree(path.toFile()); - RuleChain ruleChain = objectMapper.treeToValue(ruleChainJson.get("ruleChain"), RuleChain.class); - RuleChainMetaData ruleChainMetaData = objectMapper.treeToValue(ruleChainJson.get("metadata"), RuleChainMetaData.class); - - ruleChain.setTenantId(tenantId); - if (ruleChain.getName().equals("Root Rule Chain")) { - ruleChain.setRoot(true); - } - ruleChain = ruleChainService.saveRuleChain(ruleChain); - - ruleChainMetaData.setRuleChainId(ruleChain.getId()); - ruleChainService.saveRuleChainMetaData(new TenantId(EntityId.NULL_UUID), ruleChainMetaData); - } catch (Exception e) { - log.error("Unable to load dashboard from json: [{}]", path.toString()); - throw new RuntimeException("Unable to load dashboard from json", e); - } - } - ); - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 8d1f9d662a..fe2e7e8071 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -46,7 +46,7 @@ import java.util.concurrent.atomic.AtomicInteger; @Service public class RemoteJsInvokeService extends AbstractJsInvokeService { - @Value("${js.remote.max_requests_timeout}") + @Value("${queue.js.max_requests_timeout}") private long maxRequestsTimeout; @Getter diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c36356b1cb..9774346b3d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -59,8 +59,7 @@ services: max-size: "200m" max-file: "30" environment: - TB_HOST: tb1 - CLUSTER_NODE_ID: tb1 + TB_SERVICE_ID: tb1 env_file: - tb-node.env volumes: @@ -81,8 +80,7 @@ services: max-size: "200m" max-file: "30" environment: - TB_HOST: tb2 - CLUSTER_NODE_ID: tb2 + TB_SERVICE_ID: tb2 env_file: - tb-node.env volumes: @@ -98,8 +96,7 @@ services: ports: - "1883" environment: - TB_HOST: tb-mqtt-transport1 - CLUSTER_NODE_ID: tb-mqtt-transport1 + TB_SERVICE_ID: tb-mqtt-transport1 env_file: - tb-mqtt-transport.env volumes: @@ -113,8 +110,7 @@ services: ports: - "1883" environment: - TB_HOST: tb-mqtt-transport2 - CLUSTER_NODE_ID: tb-mqtt-transport2 + TB_SERVICE_ID: tb-mqtt-transport2 env_file: - tb-mqtt-transport.env volumes: @@ -128,8 +124,7 @@ services: ports: - "8081" environment: - TB_HOST: tb-http-transport1 - CLUSTER_NODE_ID: tb-http-transport1 + TB_SERVICE_ID: tb-http-transport1 env_file: - tb-http-transport.env volumes: @@ -143,8 +138,7 @@ services: ports: - "8081" environment: - TB_HOST: tb-http-transport2 - CLUSTER_NODE_ID: tb-http-transport2 + TB_SERVICE_ID: tb-http-transport2 env_file: - tb-http-transport.env volumes: @@ -158,8 +152,7 @@ services: ports: - "5683:5683/udp" environment: - TB_HOST: tb-coap-transport - CLUSTER_NODE_ID: tb-coap-transport + TB_SERVICE_ID: tb-coap-transport env_file: - tb-coap-transport.env volumes: diff --git a/docker/kafka.env b/docker/kafka.env index 485b3c0169..81c01dae93 100644 --- a/docker/kafka.env +++ b/docker/kafka.env @@ -4,7 +4,7 @@ KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE -KAFKA_CREATE_TOPICS=js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 +KAFKA_CREATE_TOPICS=js_eval.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 KAFKA_AUTO_CREATE_TOPICS_ENABLE=false KAFKA_LOG_RETENTION_BYTES=1073741824 KAFKA_LOG_SEGMENT_BYTES=268435456 diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index ed8f78d116..af95bc904d 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -3,4 +3,5 @@ COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 COAP_TIMEOUT=10000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index c8363230a2..71bc434241 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -3,4 +3,5 @@ HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 HTTP_REQUEST_TIMEOUT=60000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index b024c7a1a5..163cfab451 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -3,4 +3,5 @@ MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 MQTT_TIMEOUT=10000 +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index 963943dccd..aec55e9205 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -2,9 +2,9 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 -RPC_HOST=${TB_HOST} +TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 -JS_EVALUATOR=remote +JS_EVALUATOR=local TRANSPORT_TYPE=remote CACHE_TYPE=redis REDIS_HOST=redis diff --git a/docker/tb-node/conf/logback.xml b/docker/tb-node/conf/logback.xml index 4eb8a6a44f..bc694d704d 100644 --- a/docker/tb-node/conf/logback.xml +++ b/docker/tb-node/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/thingsboard/${TB_HOST}/thingsboard.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.log - /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-node/conf/thingsboard.conf b/docker/tb-node/conf/thingsboard.conf index 392b3302fd..3f27bf2859 100644 --- a/docker/tb-node/conf/thingsboard.conf +++ b/docker/tb-node/conf/thingsboard.conf @@ -15,7 +15,7 @@ # export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/coap/conf/logback.xml b/docker/tb-transports/coap/conf/logback.xml index f1ad75fd5e..20d4b6bc92 100644 --- a/docker/tb-transports/coap/conf/logback.xml +++ b/docker/tb-transports/coap/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.log - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/coap/conf/tb-coap-transport.conf b/docker/tb-transports/coap/conf/tb-coap-transport.conf index 7695e52d44..a64a5f4828 100644 --- a/docker/tb-transports/coap/conf/tb-coap-transport.conf +++ b/docker/tb-transports/coap/conf/tb-coap-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/http/conf/logback.xml b/docker/tb-transports/http/conf/logback.xml index 0aaed3ca3b..67d15ce607 100644 --- a/docker/tb-transports/http/conf/logback.xml +++ b/docker/tb-transports/http/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.log - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/http/conf/tb-http-transport.conf b/docker/tb-transports/http/conf/tb-http-transport.conf index b52f5a0f28..67d217e6c3 100644 --- a/docker/tb-transports/http/conf/tb-http-transport.conf +++ b/docker/tb-transports/http/conf/tb-http-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/docker/tb-transports/mqtt/conf/logback.xml b/docker/tb-transports/mqtt/conf/logback.xml index 4a4e19d00e..494759aeb9 100644 --- a/docker/tb-transports/mqtt/conf/logback.xml +++ b/docker/tb-transports/mqtt/conf/logback.xml @@ -21,10 +21,10 @@ - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.log - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf b/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf index 4081d5a194..4fa2550acf 100644 --- a/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf +++ b/docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf @@ -14,7 +14,7 @@ # limitations under the License. # -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" diff --git a/k8s/tb-coap-transport-configmap.yml b/k8s/tb-coap-transport-configmap.yml index d2ef3fbb47..55adbcbdec 100644 --- a/k8s/tb-coap-transport-configmap.yml +++ b/k8s/tb-coap-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-coap-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.log - /var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-coap-transport/${TB_SERVICE_ID}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-http-transport-configmap.yml b/k8s/tb-http-transport-configmap.yml index f406fd0d4f..de20a90952 100644 --- a/k8s/tb-http-transport-configmap.yml +++ b/k8s/tb-http-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-http-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.log - /var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-http-transport/${TB_SERVICE_ID}/tb-http-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-mqtt-transport-configmap.yml b/k8s/tb-mqtt-transport-configmap.yml index cf9b2c655e..9d56f6fa71 100644 --- a/k8s/tb-mqtt-transport-configmap.yml +++ b/k8s/tb-mqtt-transport-configmap.yml @@ -23,7 +23,7 @@ metadata: name: tb-mqtt-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -36,10 +36,10 @@ data: - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.log - /var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log + /var/log/tb-mqtt-transport/${TB_SERVICE_ID}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/tb-node-configmap.yml b/k8s/tb-node-configmap.yml index 7fa9612600..1f6f1abf97 100644 --- a/k8s/tb-node-configmap.yml +++ b/k8s/tb-node-configmap.yml @@ -24,7 +24,7 @@ metadata: data: conf: | export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" @@ -37,10 +37,10 @@ data: - /var/log/thingsboard/${TB_HOST}/thingsboard.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.log - /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_SERVICE_ID}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/k8s/thingsboard.yml b/k8s/thingsboard.yml index 13fadf7fe2..8f50f34f8e 100644 --- a/k8s/thingsboard.yml +++ b/k8s/thingsboard.yml @@ -247,18 +247,12 @@ spec: - containerPort: 9001 name: rpc env: - - name: RPC_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: ZOOKEEPER_ENABLED value: "true" - name: ZOOKEEPER_URL @@ -334,14 +328,12 @@ spec: - containerPort: 1883 name: mqtt env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: MQTT_BIND_ADDRESS value: "0.0.0.0" - name: MQTT_BIND_PORT @@ -409,14 +401,12 @@ spec: - containerPort: 8080 name: http env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: HTTP_BIND_ADDRESS value: "0.0.0.0" - name: HTTP_BIND_PORT @@ -484,14 +474,12 @@ spec: name: coap protocol: UDP env: - - name: CLUSTER_NODE_ID - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: TB_HOST + - name: TB_SERVICE_ID valueFrom: fieldRef: fieldPath: metadata.name + - name: TB_SERVICE_TYPE + value: "monolith" - name: COAP_BIND_ADDRESS value: "0.0.0.0" - name: COAP_BIND_PORT diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..8ff890e476 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -42,7 +42,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..4cd67df020 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -43,7 +43,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..b4af9547c8 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -73,7 +73,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka or aws-sqs or pubsub or service-bus or rabbitmq kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" From b5bedbfdf6e99ee133bad22011af4299a8d38854 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 14:54:25 +0300 Subject: [PATCH 109/196] Refactoring of Docker Compose --- .../demo/rule_chains/root_rule_chain.json | 2 +- .../controller/RuleChainController.java | 14 ++--- .../server/dao/rule/BaseRuleChainService.java | 25 +++++--- docker/README.md | 2 +- docker/docker-compose.cassandra.yml | 18 +++++- docker/docker-compose.postgres.volumes.yml | 10 +++- docker/docker-compose.postgres.yml | 18 +++++- docker/docker-compose.yml | 58 +++++++++++++++++-- docker/docker-install-tb.sh | 2 +- docker/docker-upgrade-tb.sh | 4 +- docker/haproxy/config/haproxy.cfg | 4 +- docker/tb-js-executor.env | 2 +- docker/tb-node.env | 2 +- .../server/msa/ThingsBoardDbInstaller.java | 2 +- 14 files changed, 125 insertions(+), 38 deletions(-) diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 1b321b1e1e..b68d070861 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -3,7 +3,7 @@ "additionalInfo": null, "name": "Root Rule Chain", "firstRuleNodeId": null, - "root": false, + "root": true, "debugMode": false, "configuration": null }, diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 5685947c9a..0bc518e48c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -161,15 +161,15 @@ public class RuleChainController extends BaseController { TenantId tenantId = getCurrentUser().getTenantId(); RuleChain previousRootRuleChain = ruleChainService.getRootTenantRuleChain(tenantId); if (ruleChainService.setRootRuleChain(getTenantId(), ruleChainId)) { + if (previousRootRuleChain != null) { + previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - previousRootRuleChain = ruleChainService.findRuleChainById(getTenantId(), previousRootRuleChain.getId()); - - tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), - ComponentLifecycleEvent.UPDATED); - - logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, - null, ActionType.UPDATED, null); + tbClusterService.onEntityStateChange(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), + ComponentLifecycleEvent.UPDATED); + logEntityAction(previousRootRuleChain.getId(), previousRootRuleChain, + null, ActionType.UPDATED, null); + } ruleChain = ruleChainService.findRuleChainById(getTenantId(), ruleChainId); tbClusterService.onEntityStateChange(ruleChain.getTenantId(), ruleChain.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 8ed968a78a..fdd99f97f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -88,26 +88,33 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleChain ruleChain = ruleChainDao.findById(tenantId, ruleChainId.getId()); if (!ruleChain.isRoot()) { RuleChain previousRootRuleChain = getRootTenantRuleChain(ruleChain.getTenantId()); - if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { - try { + try { + if (previousRootRuleChain == null) { + setRootAndSave(tenantId, ruleChain); + return true; + } else if (!previousRootRuleChain.getId().equals(ruleChain.getId())) { deleteRelation(tenantId, new EntityRelation(previousRootRuleChain.getTenantId(), previousRootRuleChain.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); previousRootRuleChain.setRoot(false); ruleChainDao.save(tenantId, previousRootRuleChain); - createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); - ruleChain.setRoot(true); - ruleChainDao.save(tenantId, ruleChain); + setRootAndSave(tenantId, ruleChain); return true; - } catch (ExecutionException | InterruptedException e) { - log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId); - throw new RuntimeException(e); } + } catch (ExecutionException | InterruptedException e) { + log.warn("[{}] Failed to set root rule chain, ruleChainId: [{}]", ruleChainId); + throw new RuntimeException(e); } } return false; } + private void setRootAndSave(TenantId tenantId, RuleChain ruleChain) throws ExecutionException, InterruptedException { + createRelation(tenantId, new EntityRelation(ruleChain.getTenantId(), ruleChain.getId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); + ruleChain.setRoot(true); + ruleChainDao.save(tenantId, ruleChain); + } + @Override public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData) { Validator.validateId(ruleChainMetaData.getRuleChainId(), "Incorrect rule chain id."); diff --git a/docker/README.md b/docker/README.md index ff61c2599b..9436386010 100644 --- a/docker/README.md +++ b/docker/README.md @@ -58,7 +58,7 @@ In case of any issues you can examine service logs for errors. For example to see ThingsBoard node logs execute the following command: ` -$ docker-compose logs -f tb1 +$ docker-compose logs -f tb-core1 tb-rule-engine1 ` Or use `docker-compose ps` to see the state of all the containers. diff --git a/docker/docker-compose.cassandra.yml b/docker/docker-compose.cassandra.yml index 49de5f396c..1066570140 100644 --- a/docker/docker-compose.cassandra.yml +++ b/docker/docker-compose.cassandra.yml @@ -24,14 +24,28 @@ services: - "9042" volumes: - ./tb-node/cassandra:/var/lib/cassandra - tb1: + tb-core1: env_file: - tb-node.cassandra.env depends_on: - kafka - redis - cassandra - tb2: + tb-core2: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis + - cassandra + tb-rule-engine1: + env_file: + - tb-node.cassandra.env + depends_on: + - kafka + - redis + - cassandra + tb-rule-engine2: env_file: - tb-node.cassandra.env depends_on: diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 1331393044..caaacf96e8 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,10 +20,16 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb1: + tb-core1: volumes: - tb-log-volume:/var/log/thingsboard - tb2: + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: volumes: - tb-log-volume:/var/log/thingsboard tb-coap-transport: diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index d47cbd6c37..3d19e5f968 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -27,14 +27,28 @@ services: POSTGRES_PASSWORD: postgres volumes: - ./tb-node/postgres:/var/lib/postgresql/data - tb1: + tb-core1: env_file: - tb-node.postgres.env depends_on: - kafka - redis - postgres - tb2: + tb-core2: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis + - postgres + tb-rule-engine1: + env_file: + - tb-node.postgres.env + depends_on: + - kafka + - redis + - postgres + tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9774346b3d..783a5ade46 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -48,7 +48,7 @@ services: - tb-js-executor.env depends_on: - kafka - tb1: + tb-core1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -59,7 +59,8 @@ services: max-size: "200m" max-file: "30" environment: - TB_SERVICE_ID: tb1 + TB_SERVICE_ID: tb-core1 + TB_SERVICE_TYPE: tb-core env_file: - tb-node.env volumes: @@ -69,7 +70,7 @@ services: - kafka - redis - tb-js-executor - tb2: + tb-core2: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" ports: @@ -80,7 +81,52 @@ services: max-size: "200m" max-file: "30" environment: - TB_SERVICE_ID: tb2 + TB_SERVICE_ID: tb-core2 + TB_SERVICE_TYPE: tb-core + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis + - tb-js-executor + tb-rule-engine1: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_SERVICE_ID: tb-rule-engine1 + TB_SERVICE_TYPE: tb-rule-engine + env_file: + - tb-node.env + volumes: + - ./tb-node/conf:/config + - ./tb-node/log:/var/log/thingsboard + depends_on: + - kafka + - redis + - tb-js-executor + tb-rule-engine2: + restart: always + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8080" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "30" + environment: + TB_SERVICE_ID: tb-rule-engine2 + TB_SERVICE_TYPE: tb-rule-engine env_file: - tb-node.env volumes: @@ -195,8 +241,8 @@ services: MQTT_PORT: 1883 FORCE_HTTPS_REDIRECT: "false" links: - - tb1 - - tb2 + - tb-core1 + - tb-core2 - tb-web-ui1 - tb-web-ui2 - tb-mqtt-transport1 diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 1a5d75d64a..cabb3c190d 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -49,6 +49,6 @@ if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 7db7ff6172..e62dbdc438 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,8 +44,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb-core1 docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/haproxy/config/haproxy.cfg b/docker/haproxy/config/haproxy.cfg index ba17bd64d9..bc36364024 100644 --- a/docker/haproxy/config/haproxy.cfg +++ b/docker/haproxy/config/haproxy.cfg @@ -111,6 +111,6 @@ backend tb-api-backend balance leastconn option tcp-check option log-health-checks - server tbApi1 tb1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 - server tbApi2 tb2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 + server tbApi1 tb-core1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 + server tbApi2 tb-core2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 http-request set-header X-Forwarded-Port %[dst_port] diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index 38f283cd1a..0b64f43b00 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,5 +1,5 @@ -REMOTE_JS_EVAL_REQUEST_TOPIC=js.eval.requests +REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info LOG_FOLDER=logs diff --git a/docker/tb-node.env b/docker/tb-node.env index aec55e9205..542296babe 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,7 +4,7 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 -JS_EVALUATOR=local +JS_EVALUATOR=remote TRANSPORT_TYPE=remote CACHE_TYPE=redis REDIS_HOST=redis diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 8fa7ae9a3d..d661bb7476 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -92,7 +92,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); dockerCompose.invokeCompose(); } finally { From a779839081b92a995afa67427b53bdfefe34dc92 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Apr 2020 19:16:25 +0300 Subject: [PATCH 110/196] Fixed transport --- .../org/thingsboard/server/actors/ActorSystemContext.java | 3 +++ .../common/transport/service/DefaultTransportService.java | 6 ++++++ docker/tb-coap-transport.env | 2 ++ docker/tb-http-transport.env | 2 ++ docker/tb-mqtt-transport.env | 2 ++ 5 files changed, 15 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e9b68882b..28f5b188e5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -32,6 +32,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -233,6 +234,7 @@ public class ActorSystemContext { /** * The following Service will be null if we operate in tb-core mode */ + @Lazy @Autowired(required = false) @Getter private TbRuleEngineDeviceRpcService tbRuleEngineDeviceRpcService; @@ -240,6 +242,7 @@ public class ActorSystemContext { /** * The following Service will be null if we operate in tb-rule-engine mode */ + @Lazy @Autowired(required = false) @Getter private TbCoreDeviceRpcService tbCoreDeviceRpcService; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index c820167de1..3db7300bc3 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -544,6 +544,9 @@ public class DefaultTransportService implements TransportService { protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, getTenantId(sessionInfo), getDeviceId(sessionInfo)); + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Pushing to topic {} message {}", getTenantId(sessionInfo), getDeviceId(sessionInfo), tpi.getFullTopicName(), toDeviceActorMsg); + } tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(getRoutingKey(sessionInfo), ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build()), callback != null ? @@ -552,6 +555,9 @@ public class DefaultTransportService implements TransportService { protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + if (log.isTraceEnabled()) { + log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg); + } ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index af95bc904d..8a75904185 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 71bc434241..e62424290f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 163cfab451..54c35f355c 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -1,3 +1,5 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 From 4aa225babaced85e28bdf72917de1ed437b88bc7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 22 Apr 2020 10:54:59 +0300 Subject: [PATCH 111/196] added exclusions to azure dependency, and added logger level OFF to CoreMessageReceiver --- application/src/main/resources/logback.xml | 3 +++ pom.xml | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 614f85ba0e..e25fb72ccb 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -31,8 +31,11 @@ + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8876e68538..845d590faf 100755 --- a/pom.xml +++ b/pom.xml @@ -907,6 +907,12 @@ com.microsoft.azure azure-servicebus ${azure-servicebus.version} + + + com.microsoft.azure + adal4j + + org.passay From 5b8122e1b3b4fbfcd90ab063ae0231cf12dee98f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 22 Apr 2020 15:52:36 +0300 Subject: [PATCH 112/196] added --- application/src/main/conf/logback.xml | 1 + msa/tb/docker/logback.xml | 1 + transport/coap/src/main/conf/logback.xml | 2 ++ transport/coap/src/main/resources/logback.xml | 2 ++ transport/http/src/main/conf/logback.xml | 2 ++ transport/http/src/main/resources/logback.xml | 2 ++ transport/mqtt/src/main/conf/logback.xml | 2 ++ transport/mqtt/src/main/resources/logback.xml | 2 ++ 8 files changed, 14 insertions(+) diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 295073bb82..46b42182fb 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,7 @@ + diff --git a/msa/tb/docker/logback.xml b/msa/tb/docker/logback.xml index 780a2add2f..256e78436f 100644 --- a/msa/tb/docker/logback.xml +++ b/msa/tb/docker/logback.xml @@ -42,6 +42,7 @@ + diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index c953a88ba6..2f0980ffc9 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 1172a89c28..e96ad177ef 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index c953a88ba6..2f0980ffc9 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 1172a89c28..e96ad177ef 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index c953a88ba6..2f0980ffc9 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 1172a89c28..e96ad177ef 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -27,6 +27,8 @@ + + From 30e5af36274b5e20655044d38e7b0bcbc98093ea Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 24 Apr 2020 18:55:11 +0300 Subject: [PATCH 113/196] added zookeeper configs to coap and http --- .../src/main/resources/tb-coap-transport.yml | 20 ++++++++++++++----- .../src/main/resources/tb-http-transport.yml | 20 ++++++++++++++----- .../src/main/resources/tb-mqtt-transport.yml | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c3dcb76508..2d70330187 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -17,10 +17,20 @@ spring.main.web-environment: false spring.main.web-application-type: none -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # COAP server parameters transport: @@ -42,7 +52,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index baac6e977f..0b86dc026f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -20,10 +20,20 @@ server: # Server bind port port: "${HTTP_BIND_PORT:8081}" -# Clustering properties -cluster: - # Unique id for this node (autogenerated if empty) - node_id: "${CLUSTER_NODE_ID:}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # HTTP server parameters transport: @@ -43,7 +53,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e920891d66..e267016623 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -73,7 +73,7 @@ transport: max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # kafka or in-memory or aws-sqs or pubsub or service-bus + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" From ba140644c7b95f18295c5836fc6b1fe95b904ed9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 26 Apr 2020 00:57:46 +0300 Subject: [PATCH 114/196] added Azure Service Bus queue settings --- .../src/main/resources/thingsboard.yml | 6 ++ .../azure/servicebus/TbServiceBusAdmin.java | 38 ++++++++-- .../servicebus/TbServiceBusQueueConfigs.java | 71 +++++++++++++++++++ .../provider/KafkaMonolithQueueFactory.java | 20 ++++++ .../provider/KafkaTbCoreQueueFactory.java | 19 +++++ .../KafkaTbRuleEngineQueueFactory.java | 17 +++++ .../KafkaTbTransportQueueFactory.java | 18 +++++ .../ServiceBusMonolithQueueFactory.java | 63 ++++++++++++---- .../ServiceBusTbCoreQueueFactory.java | 57 +++++++++++---- .../ServiceBusTbRuleEngineQueueFactory.java | 48 ++++++++++--- .../ServiceBusTransportQueueFactory.java | 47 ++++++++---- .../src/main/resources/tb-coap-transport.yml | 6 ++ .../src/main/resources/tb-http-transport.yml | 6 ++ .../src/main/resources/tb-mqtt-transport.yml | 6 ++ 14 files changed, 368 insertions(+), 54 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3df3cb9559..efd076f6ea 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -556,6 +556,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index 336514b7b2..229d6b4244 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -16,27 +16,32 @@ package org.thingsboard.server.queue.azure.servicebus; import com.microsoft.azure.servicebus.management.ManagementClient; +import com.microsoft.azure.servicebus.management.QueueDescription; import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.time.Duration; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @Slf4j -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") public class TbServiceBusAdmin implements TbQueueAdmin { + private final String MAX_SIZE = "maxSizeInMb"; + private final String MESSAGE_TIME_TO_LIVE = "messageTimeToLiveInSec"; + private final String LOCK_DURATION = "lockDurationInSec"; + private final Map queueConfigs; private final Set queues = ConcurrentHashMap.newKeySet(); private final ManagementClient client; - public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings) { + public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings, Map queueConfigs) { + this.queueConfigs = queueConfigs; + ConnectionStringBuilder builder = new ConnectionStringBuilder( serviceBusSettings.getNamespaceName(), "queues", @@ -60,13 +65,34 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } try { - client.createQueue(topic); + QueueDescription queueDescription = new QueueDescription(topic); + setQueueConfigs(queueDescription); + + client.createQueue(queueDescription); queues.add(topic); } catch (ServiceBusException | InterruptedException e) { log.error("Failed to create queue: [{}]", topic, e); } } + private void setQueueConfigs(QueueDescription queueDescription) { + queueConfigs.forEach((confKey, confValue) -> { + switch (confKey) { + case MAX_SIZE: + queueDescription.setMaxSizeInMB(Long.parseLong(confValue)); + break; + case MESSAGE_TIME_TO_LIVE: + queueDescription.setDefaultMessageTimeToLive(Duration.ofSeconds(Long.parseLong(confValue))); + break; + case LOCK_DURATION: + queueDescription.setLockDuration(Duration.ofSeconds(Long.parseLong(confValue))); + break; + default: + log.error("Unknown config: [{}]", confKey); + } + }); + } + public void destroy() { try { client.close(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java new file mode 100644 index 0000000000..21ab455d6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.azure.servicebus; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") +public class TbServiceBusQueueConfigs { + @Value("${queue.service-bus.queue-properties.core}") + private String coreProperties; + @Value("${queue.service-bus.queue-properties.rule-engine}") + private String ruleEngineProperties; + @Value("${queue.service-bus.queue-properties.transport-api}") + private String transportApiProperties; + @Value("${queue.service-bus.queue-properties.notifications}") + private String notificationsProperties; + @Value("${queue.service-bus.queue-properties.js-executor}") + private String jsExecutorProperties; + + @Getter + private Map coreConfigs; + @Getter + private Map ruleEngineConfigs; + @Getter + private Map transportApiConfigs; + @Getter + private Map notificationsConfigs; + @Getter + private Map jsExecutorConfigs; + + @PostConstruct + private void init() { + coreConfigs = getConfigs(coreProperties); + ruleEngineConfigs = getConfigs(ruleEngineProperties); + transportApiConfigs = getConfigs(transportApiProperties); + notificationsConfigs = getConfigs(notificationsProperties); + jsExecutorConfigs = getConfigs(jsExecutorProperties); + } + + private Map getConfigs(String properties) { + Map configs = new HashMap<>(); + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + return configs; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 72c6f7b8b9..2aca5e6da1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -49,6 +49,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @@ -247,4 +248,23 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); return builder.build(); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index ca10830981..632a6e480e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -47,6 +47,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @@ -218,4 +219,22 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index bcac399a64..dc44caf3f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -45,6 +45,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @@ -190,4 +191,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); return builder.build(); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 2f7aab4779..505aa203d7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -135,4 +137,20 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.admin(notificationAdmin); return responseBuilder.build(); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 8b01c38607..3c82e18d91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -30,19 +30,25 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -54,7 +60,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public ServiceBusMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -62,7 +73,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -70,73 +81,97 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; - this.admin = admin; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getResponsesTopic()); + return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 523ac88bd1..bcb48630f9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -29,8 +29,10 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -40,6 +42,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @@ -50,7 +54,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, TbQueueCoreSettings coreSettings, @@ -58,67 +67,91 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.admin = admin; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); + this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic(), + return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 8a2f9b396b..1f492596a1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -27,8 +27,10 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -38,6 +40,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -47,55 +51,63 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin ruleEngineAdmin; + private final TbQueueAdmin jsExecutorAdmin; + private final TbQueueAdmin notificationAdmin; public ServiceBusTbRuleEngineQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbServiceBusSettings serviceBusSettings, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.serviceBusSettings = serviceBusSettings; - this.admin = admin; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); + this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, ruleEngineSettings.getTopic(), + return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, ruleEngineSettings.getTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @@ -104,4 +116,20 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (ruleEngineAdmin != null) { + ruleEngineAdmin.destroy(); + } + if (jsExecutorAdmin != null) { + jsExecutorAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index 33c5277ca5..0b5640d384 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -23,6 +23,8 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; @@ -33,6 +35,8 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import javax.annotation.PreDestroy; + @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')") @Slf4j @@ -40,37 +44,43 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueAdmin admin; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueCoreSettings coreSettings; + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin transportApiAdmin; + private final TbQueueAdmin notificationAdmin; + public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbServiceInfoProvider serviceInfoProvider, + TbQueueTransportNotificationSettings transportNotificationSettings, + TbServiceBusSettings serviceBusSettings, + TbServiceInfoProvider serviceInfoProvider, TbQueueCoreSettings coreSettings, - TbQueueAdmin admin) { + TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; - this.admin = admin; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); } @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { TbQueueProducer> producerTemplate = - new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); TbQueueConsumer> consumerTemplate = - new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(admin); + templateBuilder.queueAdmin(transportApiAdmin); templateBuilder.requestTemplate(producerTemplate); templateBuilder.responseTemplate(consumerTemplate); templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); @@ -81,18 +91,31 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, transportApiSettings.getRequestsTopic()); + return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, transportApiSettings.getRequestsTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(admin, serviceBusSettings, coreSettings.getTopic()); + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbServiceBusConsumerTemplate<>(admin, serviceBusSettings, + return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId(), msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (transportApiAdmin != null) { + transportApiAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 2d70330187..47a011c35f 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -95,6 +95,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0b86dc026f..55236dd84f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -96,6 +96,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e267016623..3872a6d0b3 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -116,6 +116,12 @@ queue: sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" From 86f21023febedb362e8f991a97886d5cee295dec Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Apr 2020 14:44:29 +0300 Subject: [PATCH 115/196] Improved PartitionService and fixed startup order in docker-compose.yml --- .../ConsistentHashParitionServiceTest.java | 21 +++++++++++++------ .../ConsistentHashPartitionService.java | 12 ++++++++++- docker/docker-compose.yml | 4 ++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java index 3f7619ed1e..6b03482c0e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/ConsistentHashParitionServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cluster.routing; import com.datastax.driver.core.utils.UUIDs; import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,6 +32,8 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import java.util.ArrayList; import java.util.Collections; @@ -41,6 +44,7 @@ import java.util.Map; import java.util.stream.Collectors; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) @@ -52,6 +56,7 @@ public class ConsistentHashParitionServiceTest { private TbServiceInfoProvider discoveryService; private TenantRoutingInfoService routingInfoService; private ApplicationEventPublisher applicationEventPublisher; + private TbQueueRuleEngineSettings ruleEngineSettings; private String hashFunctionName = "murmur3_128"; private Integer virtualNodesSize = 16; @@ -62,12 +67,15 @@ public class ConsistentHashParitionServiceTest { discoveryService = mock(TbServiceInfoProvider.class); applicationEventPublisher = mock(ApplicationEventPublisher.class); routingInfoService = mock(TenantRoutingInfoService.class); - clusterRoutingService = new ConsistentHashPartitionService(discoveryService, routingInfoService, applicationEventPublisher); + ruleEngineSettings = mock(TbQueueRuleEngineSettings.class); + clusterRoutingService = new ConsistentHashPartitionService(discoveryService, + routingInfoService, + applicationEventPublisher, + ruleEngineSettings + ); + when(ruleEngineSettings.getQueues()).thenReturn(Collections.emptyList()); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 3); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEngineTopic", "tb.rule-engine"); - ReflectionTestUtils.setField(clusterRoutingService, "ruleEnginePartitions", 100); - ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() @@ -107,8 +115,9 @@ public class ConsistentHashParitionServiceTest { List> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); long end = System.currentTimeMillis(); double diff = (data.get(data.size() - 1).getValue() - data.get(0).getValue()); - System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", (diff / ITERATIONS) * 100.0) + "%)"); - + double diffPercent = (diff / ITERATIONS) * 100.0; + System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + diff + "(" + String.format("%f", diffPercent) + "%)"); + Assert.assertTrue(diffPercent < 0.5); for (Map.Entry entry : data) { System.out.println(entry.getKey() + ": " + entry.getValue()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java index df25ea3ba9..918570bc56 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ConsistentHashPartitionService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; @@ -61,6 +62,7 @@ public class ConsistentHashPartitionService implements PartitionService { private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; + private final TbQueueRuleEngineSettings tbQueueRuleEngineSettings; private final ConcurrentMap partitionTopics = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizes = new ConcurrentHashMap<>(); private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); @@ -74,10 +76,14 @@ public class ConsistentHashPartitionService implements PartitionService { private HashFunction hashFunction; - public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, TenantRoutingInfoService tenantRoutingInfoService, ApplicationEventPublisher applicationEventPublisher) { + public ConsistentHashPartitionService(TbServiceInfoProvider serviceInfoProvider, + TenantRoutingInfoService tenantRoutingInfoService, + ApplicationEventPublisher applicationEventPublisher, + TbQueueRuleEngineSettings tbQueueRuleEngineSettings) { this.serviceInfoProvider = serviceInfoProvider; this.tenantRoutingInfoService = tenantRoutingInfoService; this.applicationEventPublisher = applicationEventPublisher; + this.tbQueueRuleEngineSettings = tbQueueRuleEngineSettings; } @PostConstruct @@ -85,6 +91,10 @@ public class ConsistentHashPartitionService implements PartitionService { this.hashFunction = forName(hashFunctionName); partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions); partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic); + tbQueueRuleEngineSettings.getQueues().forEach(queueConfiguration -> { + partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getTopic()); + partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getPartitions()); + }); } @Override diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 783a5ade46..2f0d0fb3b3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -70,6 +70,8 @@ services: - kafka - redis - tb-js-executor + - tb-rule-engine1 + - tb-rule-engine2 tb-core2: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" @@ -92,6 +94,8 @@ services: - kafka - redis - tb-js-executor + - tb-rule-engine1 + - tb-rule-engine2 tb-rule-engine1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" From cb6e91277f8dad93a2f09d2f1d3efce35e9cd9e0 Mon Sep 17 00:00:00 2001 From: bdrlamb Date: Mon, 27 Apr 2020 14:32:35 -0700 Subject: [PATCH 116/196] Fix MQTT inactivity disconnects MQTT clients don't send PINGREQ messages if they've recently sent [UN]SUBSCRIBE/PUBLISH messages. The MQTT server last activity timestamp is currently only updated in response to PINGREQ messages. This causes the server to disconnect any clients that regularly send PUBLISH messages since they don't send PINGREQ. This change adds reportActivity calls in response to PUBLISH, SUBSCRIBE and UNSUBSCRIBE in addition to PINGREQ. --- .../server/transport/mqtt/MqttTransportHandler.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 7295cb786b..d0f9c489cc 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -138,12 +138,24 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement break; case PUBLISH: processPublish(ctx, (MqttPublishMessage) msg); + transportService.reportActivity(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.reportActivity(); + } break; case SUBSCRIBE: processSubscribe(ctx, (MqttSubscribeMessage) msg); + transportService.reportActivity(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.reportActivity(); + } break; case UNSUBSCRIBE: processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg); + transportService.reportActivity(sessionInfo); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.reportActivity(); + } break; case PINGREQ: if (checkConnected(ctx, msg)) { From 0dc31fbdde07dada1e79e8d7ce1c57cab97324b9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Apr 2020 16:13:16 +0300 Subject: [PATCH 117/196] created awsSqs, pubSub, rabbitmq js-executors --- .../provider/AwsSqsMonolithQueueFactory.java | 36 +++- .../provider/PubSubMonolithQueueFactory.java | 36 +++- .../RabbitMqMonolithQueueFactory.java | 37 +++- .../api/jsInvokeMessageProcessor.js | 59 +++---- .../config/custom-environment-variables.yml | 23 ++- msa/js-executor/config/default.yml | 14 +- msa/js-executor/package.json | 3 + msa/js-executor/queue/awsSqsTemplate.js | 163 ++++++++++++++++++ .../queue/{kafka => }/kafkaTemplate.js | 61 ++++--- msa/js-executor/queue/pubSubTemplate.js | 87 ++++++++++ msa/js-executor/queue/rabbitmqTemplate.js | 149 ++++++++++++++++ msa/js-executor/server.js | 24 ++- 12 files changed, 616 insertions(+), 76 deletions(-) create mode 100644 msa/js-executor/queue/awsSqsTemplate.js rename msa/js-executor/queue/{kafka => }/kafkaTemplate.js (64%) create mode 100644 msa/js-executor/queue/pubSubTemplate.js create mode 100644 msa/js-executor/queue/rabbitmqTemplate.js diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 76ff04c238..c86baf9c48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -15,20 +15,26 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.js.JsInvokeProtos.*; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -40,6 +46,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") @@ -52,6 +59,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -65,7 +73,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { + TbAwsSqsQueueAttributes sqsQueueAttributes, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -73,6 +82,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -144,8 +154,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> { + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index b1afc61dbd..760fa7708d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -30,6 +33,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -39,11 +43,14 @@ import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -56,6 +63,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueAdmin admin; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -63,7 +71,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, PartitionService partitionService, - TbServiceInfoProvider serviceInfoProvider) { + TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -72,6 +81,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.admin = new TbPubSubAdmin(pubSubSettings); this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -138,7 +148,25 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(admin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(admin, pubSubSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(admin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index ff4a69e2e6..2435b30851 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -15,28 +15,37 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import java.nio.charset.StandardCharsets; + @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { @@ -48,6 +57,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueAdmin admin; public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, @@ -56,6 +67,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueAdmin admin) { this.partitionService = partitionService; this.coreSettings = coreSettings; @@ -64,6 +76,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; + this.jsInvokeSettings = jsInvokeSettings; this.admin = admin; } @@ -130,7 +143,25 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(admin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(admin, rabbitMqSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(admin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } } diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 4cf20b4227..f6da6886e7 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,7 +19,6 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; -let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -31,7 +30,7 @@ const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); function JsInvokeMessageProcessor(producer) { - console.log("Kafka Producer:", producer); + console.log("Producer:", producer); this.producer = producer; this.executor = new JsExecutor(useSandbox); this.scriptMap = {}; @@ -39,26 +38,27 @@ function JsInvokeMessageProcessor(producer) { this.executedScriptsCounter = 0; } -JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { +JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(messageStr) { - var requestId; - var responseTopic; + let requestId; + let responseTopic; try { - var request = JSON.parse(message.value.toString('utf8')); - headers = message.headers; - var buf = message.headers['requestId']; + let message = JSON.parse(messageStr); + let request = JSON.parse(Buffer.from(message.data).toString('utf8')); + let headers = message.headers; + let buf = Buffer.from(headers.data['requestId']); requestId = Utils.UUIDFromBuffer(buf); - buf = message.headers['responseTopic']; + buf = Buffer.from(headers.data['responseTopic']); responseTopic = buf.toString('utf8'); logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic); if (request.compileRequest) { - this.processCompileRequest(requestId, responseTopic, request.compileRequest); + this.processCompileRequest(requestId, responseTopic, headers, request.compileRequest); } else if (request.invokeRequest) { - this.processInvokeRequest(requestId, responseTopic, request.invokeRequest); + this.processInvokeRequest(requestId, responseTopic, headers, request.invokeRequest); } else if (request.releaseRequest) { - this.processReleaseRequest(requestId, responseTopic, request.releaseRequest); + this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest); } else { logger.error('[%s] Unknown request recevied!', requestId); } @@ -69,7 +69,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { } } -JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, compileRequest) { +JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, headers, compileRequest) { var scriptId = getScriptId(compileRequest); logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId); @@ -78,17 +78,17 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r this.cacheScript(scriptId, script); var compileResponse = createCompileResponse(scriptId, true); logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, compileResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse); }, (err) => { var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, compileResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse); } ); } -JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, invokeRequest) { +JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, headers, invokeRequest) { var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; @@ -104,7 +104,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re (result) => { var invokeResponse = createInvokeResponse(result, true); logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); }, (err) => { var errorCode; @@ -115,19 +115,19 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re } var invokeResponse = createInvokeResponse("", false, errorCode, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode); - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); } ) }, (err) => { var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR); - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); } ); } -JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, releaseRequest) { +JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); if (this.scriptMap[scriptId]) { @@ -139,28 +139,17 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r } var releaseResponse = createReleaseResponse(scriptId, true); logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, null, null, releaseResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, null, releaseResponse); } -JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) { +JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, headers, scriptId, compileResponse, invokeResponse, releaseResponse) { var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); - this.producer.send(responseTopic, scriptId, rawResponse, headers - // { - // topic: responseTopic, - // messages: [ - // { - // key: scriptId, - // value: rawResponse, - // headers: headers - // } - // ] - // } - ).then( + this.producer.send(responseTopic, scriptId, rawResponse, headers).then( () => {}, (err) => { if (err) { - logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message); + logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); logger.error(err.stack); } } diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 77e68b0429..3beab91b40 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -15,12 +15,33 @@ # service-type: "TB_SERVICE_TYPE" +request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" kafka: - request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" bootstrap: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" + +pubsub: + project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" + service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" + +aws_sqs: + access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" + secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" + region: "TB_QUEUE_AWS_SQS_REGION" + +rabbitmq: + exchange_name: "TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME" + host: "TB_QUEUE_RABBIT_MQ_HOST" + port: "TB_QUEUE_RABBIT_MQ_PORT" + virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" + username: "TB_QUEUE_RABBIT_MQ_USERNAME" + password: "TB_QUEUE_RABBIT_MQ_PASSWORD" + automatic_recovery_enabled: "TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED" + connection_timeout: "TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT" + handshake_timeout: "TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT" + logger: level: "LOGGER_LEVEL" path: "LOG_FOLDER" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 724901d171..ee38296b31 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -15,13 +15,25 @@ # service-type: "kafka" +request_topic: "js.eval.requests" kafka: - request_topic: "js.eval.requests" bootstrap: # Kafka Bootstrap Servers servers: "localhost:9092" +rabbitmq: + exchange_name: "" + host: "localhost" + port: "5672" + virtual_host: "/" + username: "YOUR_USERNAME" + password: "YOUR_PASSWORD" + automatic_recovery_enabled: "false" + connection_timeout: "60000" + handshake_timeout: "10000" + + logger: level: "info" path: "logs" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index c87b62c6eb..6b2ac41a9c 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -15,6 +15,9 @@ "config": "^3.2.2", "js-yaml": "^3.12.0", "kafkajs": "^1.11.0", + "@google-cloud/pubsub": "^1.7.1", + "aws-sdk": "^2.663.0", + "amqplib": "^0.5.5", "long": "^4.0.0", "uuid-parse": "^1.0.0", "winston": "^3.0.0", diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js new file mode 100644 index 0000000000..6bd2f73510 --- /dev/null +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -0,0 +1,163 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('awsSqsTemplate'); + +const requestTopic = config.get('request_topic'); + +const accessKeyId = config.get('aws_sqs.access_key_id'); +const secretAccessKey = config.get('aws_sqs.secret_access_key'); +const region = config.get('aws_sqs.region'); +const AWS = require('aws-sdk'); + +let sqsClient; +let queueURL; +let responseTopics = new Map(); +let stopped = false; + +function AwsSqsProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + let msgBody = JSON.stringify( + { + key: scriptId, + data: [...rawResponse], + headers: headers + }); + + let responseQueueUrl = responseTopics.get(responseTopic); + + if (!responseQueueUrl) { + responseQueueUrl = await createQueue(responseTopic); + responseTopics.set(responseTopic, responseQueueUrl); + } + + let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; + + return new Promise((resolve, reject) => { + sqsClient.sendMessage(params, function (err, data) { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + AWS.config.update({accessKeyId: accessKeyId, secretAccessKey: secretAccessKey, region: region}); + + sqsClient = new AWS.SQS({apiVersion: '2012-11-05'}); + + queueURL = await createQueue(requestTopic); + const messageProcessor = new JsInvokeMessageProcessor(new AwsSqsProducer()); + + const params = { + MaxNumberOfMessages: 10, + QueueUrl: queueURL, + WaitTimeSeconds: 0.025 + }; + while (!stopped) { + const messages = await new Promise((resolve, reject) => { + sqsClient.receiveMessage(params, function (err, data) { + if (err) { + reject(err); + } else { + resolve(data.Messages); + } + }); + }); + + if (messages && messages.length > 0) { + const entries = []; + + messages.forEach(message => { + entries.push({ + Id: message.MessageId, + ReceiptHandle: message.ReceiptHandle + }); + messageProcessor.onJsInvokeMessage(message.Body); + }); + + const deleteBatch = { + QueueUrl: queueURL, + Entries: entries + }; + sqsClient.deleteMessageBatch(deleteBatch, function (err, data) { + if (err) { + logger.error("Failed to delete messages from queue.", err.message); + } else { + //do nothing + } + }); + } + } + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +function createQueue(topic) { + let queueName = topic.replace(/\./g, '_') + '.fifo'; + let queueParams = { + QueueName: queueName, Attributes: { + FifoQueue: 'true', + ContentBasedDeduplication: 'true' + + } + }; + return new Promise((resolve, reject) => { + sqsClient.createQueue(queueParams, function (err, data) { + if (err) { + reject(err); + } else { + resolve(data.QueueUrl); + } + }); + }); +} + +process.on('exit', () => { + stopped = true; + logger.info('Aws Sqs client stopped.'); + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + if (sqsClient) { + logger.info('Stopping Aws Sqs client.') + try { + await sqsClient.close(); + logger.info('Aws Sqs client is stopped.') + process.exit(status); + } catch (e) { + logger.info('Aws Sqs client stop error.'); + process.exit(status); + } + } else { + process.exit(status); + } +} diff --git a/msa/js-executor/queue/kafka/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js similarity index 64% rename from msa/js-executor/queue/kafka/kafkaTemplate.js rename to msa/js-executor/queue/kafkaTemplate.js index bb3f81c48d..38e713a7db 100644 --- a/msa/js-executor/queue/kafka/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -13,39 +13,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const { logLevel, Kafka } = require('kafkajs'); +const {logLevel, Kafka} = require('kafkajs'); const config = require('config'), - JsInvokeMessageProcessor = require('../../api/jsInvokeMessageProcessor'), - logger = require('../../config/logger')._logger('main'), - KafkaJsWinstonLogCreator = require('../../config/logger').KafkaJsWinstonLogCreator; + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('kafkaTemplate'), + KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; -var kafkaClient; -var consumer; -var producer; +let kafkaClient; +let consumer; +let producer; function KafkaProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { + let headersData = headers.data; + headersData = Object.fromEntries(Object.entries(headersData).map(([key, value]) => [key, Buffer.from(value)])); return producer.send( - { - topic: responseTopic, - messages: [ - { - key: scriptId, - value: rawResponse, - headers: headers - } - ] - }); + { + topic: responseTopic, + messages: [ + { + key: scriptId, + value: rawResponse, + headers: headersData + } + ] + }); } } -(async() => { +(async () => { try { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); - const kafkaRequestTopic = config.get('kafka.request_topic'); + const kafkaRequestTopic = config.get('request_topic'); logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); @@ -56,17 +58,28 @@ function KafkaProducer() { logCreator: KafkaJsWinstonLogCreator }); - consumer = kafkaClient.consumer({ groupId: 'js-executor-group' }); + consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); producer = kafkaClient.producer(); const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); - await consumer.subscribe({ topic: kafkaRequestTopic}); + await consumer.subscribe({topic: kafkaRequestTopic}); logger.info('Started ThingsBoard JavaScript Executor Microservice.'); await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - messageProcessor.onJsInvokeMessage(message); + eachMessage: async ({topic, partition, message}) => { + let headers = message.headers; + let key = message.key; + let data = message.value; + let msg = {}; + + headers = Object.fromEntries( + Object.entries(headers).map(([key, value]) => [key, [...value]])); + + msg.key = key.toString('utf8'); + msg.data = [...data]; + msg.headers = {data: headers} + messageProcessor.onJsInvokeMessage(JSON.stringify(msg)); }, }); @@ -85,7 +98,7 @@ async function exit(status) { logger.info('Exiting with status: %d ...', status); if (consumer) { logger.info('Stopping Kafka Consumer...'); - var _consumer = consumer; + let _consumer = consumer; consumer = null; try { await _consumer.disconnect(); diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js new file mode 100644 index 0000000000..c8b39f7d6b --- /dev/null +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -0,0 +1,87 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('pubSubTemplate'); +const {PubSub} = require('@google-cloud/pubsub'); + +const projectId = config.get('pubsub.project_id'); +const credentials = JSON.parse(config.get('pubsub.service_account')); +const requestTopic = config.get('request_topic'); + +let pubSubClient; + +function PubSubProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + let data = JSON.stringify( + { + key: scriptId, + data: [...rawResponse], + headers: headers + }); + let dataBuffer = Buffer.from(data); + return pubSubClient.topic(responseTopic).publish(dataBuffer); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + pubSubClient = new PubSub({projectId: projectId, credentials: credentials}); + + const subscription = pubSubClient.subscription(requestTopic); + + const messageProcessor = new JsInvokeMessageProcessor(new PubSubProducer()); + + const messageHandler = message => { + + messageProcessor.onJsInvokeMessage(message.data.toString('utf8')); + message.ack(); + }; + + subscription.on('message', messageHandler); + + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + if (pubSubClient) { + logger.info('Stopping Pub/Sub client.') + try { + await pubSubClient.close(); + logger.info('Pub/Sub client is stopped.') + process.exit(status); + } catch (e) { + logger.info('Pub/Sub client stop error.'); + process.exit(status); + } + } else { + process.exit(status); + } +} + diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js new file mode 100644 index 0000000000..0b48cdd62f --- /dev/null +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -0,0 +1,149 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('rabbitmqTemplate'); + +const requestTopic = config.get('request_topic'); +const amqp = require('amqplib/callback_api'); +let connection; +let channel; +let stopped = false; +const responseTopics = []; + +function RabbitMqProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + + if (!responseTopics.includes(responseTopic)) { + await createQueue(responseTopic); + responseTopics.push(responseTopic); + } + + let data = JSON.stringify( + { + key: scriptId, + data: [...rawResponse], + headers: headers + }); + let dataBuffer = Buffer.from(data); + channel.sendToQueue(responseTopic, dataBuffer); + return new Promise((resolve, reject) => { + channel.waitForConfirms((err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + + amqp.credentials.amqplain('admin', 'password'); + connection = await new Promise((resolve, reject) => { + amqp.connect('amqp://localhost:5672/', function (err, connection) { + if (err) { + reject(err); + } else { + resolve(connection); + } + }); + }); + + channel = await new Promise((resolve, reject) => { + connection.createConfirmChannel(function (err, channel) { + if (err) { + reject(err); + } else { + resolve(channel); + } + }); + }); + + await createQueue(requestTopic); + + const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer()); + + while (!stopped) { + let message = await new Promise((resolve, reject) => { + channel.get(requestTopic, {}, function (err, msg) { + if (err) { + reject(err); + } else { + resolve(msg); + } + }); + }); + + if (message) { + messageProcessor.onJsInvokeMessage(message.content.toString('utf8')); + channel.ack(message); + } + } + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +function createQueue(topic) { + let params = {durable: false}; + return new Promise((resolve, reject) => { + channel.assertQueue(topic, params, function (err, data) { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +} + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + + if (channel) { + logger.info('Stopping RabbitMq chanel.') + await channel.close(); + logger.info('RabbitMq chanel is stopped'); + } + + if (connection) { + logger.info('Stopping RabbitMq connection.') + try { + await connection.close(); + logger.info('RabbitMq client is connection.') + process.exit(status); + } catch (e) { + logger.info('RabbitMq connection stop error.'); + process.exit(status); + } + } else { + process.exit(status); + } +} \ No newline at end of file diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 486f9dc187..bd84289200 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -14,16 +14,32 @@ * limitations under the License. */ -const config = require('config'); +const config = require('config'), logger = require('./config/logger')._logger('main'); const serviceType = config.get('service-type'); switch (serviceType) { case 'kafka': - require('./queue/kafka/kafkaTemplate'); - console.log('Used kafka template.'); + logger.info('Starting kafka template.'); + require('./queue/kafkaTemplate'); + logger.info('kafka template is started.'); + break; + case 'pubsub': + logger.info('Starting Pub/Sub template.') + require('./queue/pubSubTemplate'); + logger.info('Pub/Sub template is started.') + break; + case 'aws-sqs': + logger.info('Starting Aws Sqs template.') + require('./queue/awsSqsTemplate'); + logger.info('Aws Sqs template is started.') + break; + case 'rabbitmq': + logger.info('Starting RabbitMq template.') + require('./queue/rabbitmqTemplate'); + logger.info('RabbitMq template is started.') break; default: - console.error('Unknown service type: ', serviceType); + logger.error('Unknown service type: ', serviceType); process.exit(-1); } From d264a5206c35e1b9d1cbab38902c8262516db753 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Apr 2020 17:05:53 +0300 Subject: [PATCH 118/196] revert RemoteJsInvokeService --- .../server/service/script/RemoteJsInvokeService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index baf54bfe2d..fe2e7e8071 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -50,13 +50,13 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private long maxRequestsTimeout; @Getter -// @Value("${queue.js.max_errors}") + @Value("${js.remote.max_errors}") private int maxErrors; - @Value("${queue.js.max_black_list_duration_sec:60}") + @Value("${js.remote.max_black_list_duration_sec:60}") private int maxBlackListDurationSec; - @Value("${queue.js.stats.enabled:false}") + @Value("${js.remote.stats.enabled:false}") private boolean statsEnabled; private final AtomicInteger kafkaPushedMsgs = new AtomicInteger(0); @@ -65,7 +65,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final AtomicInteger kafkaFailedMsgs = new AtomicInteger(0); private final AtomicInteger kafkaTimeoutMsgs = new AtomicInteger(0); -// @Scheduled(fixedDelayString = "${queue.js.stats.print_interval_ms}") + @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") public void printStats() { if (statsEnabled) { int pushedMsgs = kafkaPushedMsgs.getAndSet(0); From 214a6060669ec1c3e68bba8ce66510a114d56bd2 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Apr 2020 17:07:25 +0300 Subject: [PATCH 119/196] revert thingsboard.yml --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 573f6b9462..bf82c586c3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -606,9 +606,9 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}" + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}" + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout From eb3cc332d187464bbe296ef5ee8ebb81ca5be85e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 28 Apr 2020 18:19:30 +0300 Subject: [PATCH 120/196] Fix gateway/device last activity time checks --- common/queue/src/main/proto/queue.proto | 2 ++ .../transport/mqtt/MqttTransportHandler.java | 33 +++++++++---------- .../mqtt/session/GatewayDeviceSessionCtx.java | 2 ++ .../mqtt/session/GatewaySessionHandler.java | 4 +-- .../service/DefaultTransportService.java | 26 ++++++++++----- pom.xml | 1 + 6 files changed, 41 insertions(+), 27 deletions(-) diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 51bc7649be..d3e850b39c 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -49,6 +49,8 @@ message SessionInfoProto { int64 deviceIdLSB = 7; string deviceName = 8; string deviceType = 9; + int64 gwSessionIdMSB = 10; + int64 gwSessionIdLSB = 11; } enum SessionEvent { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index d0f9c489cc..351be0865f 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -100,7 +100,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private volatile DeviceSessionCtx deviceSessionCtx; private volatile GatewaySessionHandler gatewaySessionHandler; - MqttTransportHandler(MqttTransportContext context,SslHandler sslHandler) { + MqttTransportHandler(MqttTransportContext context, SslHandler sslHandler) { this.sessionId = UUID.randomUUID(); this.context = context; this.transportService = context.getTransportService(); @@ -138,32 +138,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement break; case PUBLISH: processPublish(ctx, (MqttPublishMessage) msg); - transportService.reportActivity(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.reportActivity(); - } break; case SUBSCRIBE: processSubscribe(ctx, (MqttSubscribeMessage) msg); - transportService.reportActivity(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.reportActivity(); - } break; case UNSUBSCRIBE: processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg); - transportService.reportActivity(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.reportActivity(); - } break; case PINGREQ: if (checkConnected(ctx, msg)) { ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); transportService.reportActivity(sessionInfo); - if (gatewaySessionHandler != null) { - gatewaySessionHandler.reportActivity(); - } } break; case DISCONNECT: @@ -174,7 +159,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement default: break; } - } private void processPublish(ChannelHandlerContext ctx, MqttPublishMessage mqttMsg) { @@ -188,6 +172,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement if (topicName.startsWith(MqttTopics.BASE_GATEWAY_API_TOPIC)) { if (gatewaySessionHandler != null) { handleGatewayPublishMsg(topicName, msgId, mqttMsg); + transportService.reportActivity(sessionInfo); } } else { processDevicePublish(ctx, mqttMsg, topicName, msgId); @@ -244,6 +229,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } else if (topicName.equals(MqttTopics.DEVICE_CLAIM_TOPIC)) { TransportProtos.ClaimDeviceMsg claimDeviceMsg = adaptor.convertToClaimDevice(deviceSessionCtx, mqttMsg); transportService.process(sessionInfo, claimDeviceMsg, getPubAckCallback(ctx, msgId, claimDeviceMsg)); + } else { + transportService.reportActivity(sessionInfo); } } catch (AdaptorException e) { log.warn("[{}] Failed to process publish msg [{}][{}]", sessionId, topicName, msgId, e); @@ -276,6 +263,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } log.trace("[{}] Processing subscription [{}]!", sessionId, mqttMsg.variableHeader().messageId()); List grantedQoSList = new ArrayList<>(); + boolean activityReported = false; for (MqttTopicSubscription subscription : mqttMsg.payload().topicSubscriptions()) { String topic = subscription.topicName(); MqttQoS reqQoS = subscription.qualityOfService(); @@ -284,11 +272,13 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.DEVICE_ATTRIBUTES_TOPIC: { transportService.process(sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().build(), null); registerSubQoS(topic, grantedQoSList, reqQoS); + activityReported = true; break; } case MqttTopics.DEVICE_RPC_REQUESTS_SUB_TOPIC: { transportService.process(sessionInfo, TransportProtos.SubscribeToRPCMsg.newBuilder().build(), null); registerSubQoS(topic, grantedQoSList, reqQoS); + activityReported = true; break; } case MqttTopics.DEVICE_RPC_RESPONSE_SUB_TOPIC: @@ -308,6 +298,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement grantedQoSList.add(FAILURE.value()); } } + if (!activityReported) { + transportService.reportActivity(sessionInfo); + } ctx.writeAndFlush(createSubAckMessage(mqttMsg.variableHeader().messageId(), grantedQoSList)); } @@ -320,6 +313,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement if (!checkConnected(ctx, mqttMsg)) { return; } + boolean activityReported = false; log.trace("[{}] Processing subscription [{}]!", sessionId, mqttMsg.variableHeader().messageId()); for (String topicName : mqttMsg.payload().topics()) { mqttQoSMap.remove(new MqttTopicMatcher(topicName)); @@ -327,10 +321,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement switch (topicName) { case MqttTopics.DEVICE_ATTRIBUTES_TOPIC: { transportService.process(sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().setUnsubscribe(true).build(), null); + activityReported = true; break; } case MqttTopics.DEVICE_RPC_REQUESTS_SUB_TOPIC: { transportService.process(sessionInfo, TransportProtos.SubscribeToRPCMsg.newBuilder().setUnsubscribe(true).build(), null); + activityReported = true; break; } } @@ -338,6 +334,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement log.warn("[{}] Failed to process unsubscription [{}] to [{}]", sessionId, mqttMsg.variableHeader().messageId(), topicName); } } + if (!activityReported) { + transportService.reportActivity(sessionInfo); + } ctx.writeAndFlush(createUnSubAckMessage(mqttMsg.variableHeader().messageId())); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java index b4b5f98238..c137da0b82 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewayDeviceSessionCtx.java @@ -46,6 +46,8 @@ public class GatewayDeviceSessionCtx extends MqttDeviceAwareSessionContext imple .setTenantIdLSB(deviceInfo.getTenantIdLSB()) .setDeviceName(deviceInfo.getDeviceName()) .setDeviceType(deviceInfo.getDeviceType()) + .setGwSessionIdMSB(parent.getSessionId().getMostSignificantBits()) + .setGwSessionIdLSB(parent.getSessionId().getLeastSignificantBits()) .build(); setDeviceInfo(deviceInfo); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index de5f82f324..65393d6939 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -410,7 +410,7 @@ public class GatewaySessionHandler { return deviceSessionCtx.nextMsgId(); } - public void reportActivity() { - devices.forEach((id, deviceCtx) -> transportService.reportActivity(deviceCtx.getSessionInfo())); + public UUID getSessionId() { + return sessionId; } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 3db7300bc3..14f01a6d63 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -379,6 +379,7 @@ public class DefaultTransportService implements TransportService { @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback callback) { if (checkLimits(sessionInfo, msg, callback)) { + reportActivityInternal(sessionInfo); sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) .setClaimDevice(msg).build(), callback); } @@ -401,23 +402,32 @@ public class DefaultTransportService implements TransportService { private void checkInactivityAndReportActivity() { long expTime = System.currentTimeMillis() - sessionInactivityTimeout; sessions.forEach((uuid, sessionMD) -> { - if (sessionMD.getLastActivityTime() < expTime) { + long lastActivityTime = sessionMD.getLastActivityTime(); + TransportProtos.SessionInfoProto sessionInfo = sessionMD.getSessionInfo(); + if (sessionInfo.getGwSessionIdMSB() > 0 && + sessionInfo.getGwSessionIdLSB() > 0) { + SessionMetaData gwMetaData = sessions.get(new UUID(sessionInfo.getGwSessionIdMSB(), sessionInfo.getGwSessionIdLSB())); + if (gwMetaData != null) { + lastActivityTime = Math.max(gwMetaData.getLastActivityTime(), lastActivityTime); + } + } + if (lastActivityTime < expTime) { if (log.isDebugEnabled()) { - log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime()); + log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionInfo), lastActivityTime); } - process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); + process(sessionInfo, getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null); sessions.remove(uuid); sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance()); } else { - if (sessionMD.getLastActivityTime() > sessionMD.getLastReportedActivityTime()) { - final long lastActivityTime = sessionMD.getLastActivityTime(); - process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder() + if (lastActivityTime > sessionMD.getLastReportedActivityTime()) { + final long lastActivityTimeFinal = lastActivityTime; + process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() .setAttributeSubscription(sessionMD.isSubscribedToAttributes()) .setRpcSubscription(sessionMD.isSubscribedToRPC()) - .setLastActivityTime(sessionMD.getLastActivityTime()).build(), new TransportServiceCallback() { + .setLastActivityTime(lastActivityTime).build(), new TransportServiceCallback() { @Override public void onSuccess(Void msg) { - sessionMD.setLastReportedActivityTime(lastActivityTime); + sessionMD.setLastReportedActivityTime(lastActivityTimeFinal); } @Override diff --git a/pom.xml b/pom.xml index 845d590faf..494a43cb56 100755 --- a/pom.xml +++ b/pom.xml @@ -305,6 +305,7 @@ **/.env + **/*.env **/.eslintrc **/.babelrc **/.jshintrc From 6e50e9776726ce45b1fa86e8b50a42d2cbfd95b3 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Tue, 28 Apr 2020 20:05:54 +0300 Subject: [PATCH 121/196] Add termostat dashboard reletion (#2687) * Add termostat dashboard reletion * Add create reletion --- .../json/demo/dashboards/theromstats.json | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/application/src/main/data/json/demo/dashboards/theromstats.json b/application/src/main/data/json/demo/dashboards/theromstats.json index bd191a6bc5..c7ab182171 100644 --- a/application/src/main/data/json/demo/dashboards/theromstats.json +++ b/application/src/main/data/json/demo/dashboards/theromstats.json @@ -121,7 +121,7 @@ "type": "customPretty", "customHtml": "\n
\n \n
\n

Add thermostat

\n \n \n \n \n
\n \n \n
\n \n \n \n
\n
Thermostat name is required.
\n
\n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Create\n Cancel\n \n
\n", "customCss": ".add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n", - "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n saveEntityPromise().then(\n function (entity) {\n saveAttributes(entity.id);\n updateAliasData();\n $mdDialog.hide();\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}\n" + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n types = $injector.get('types'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService'),\n userService = $injector.get('userService'),\n entityRelationService = $injector.get('entityRelationService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n vm.relations = [];\n\n vm.addRelation = function() {\n var relation = {\n direction: types.entitySearchDirection.from,\n relationType: 'Contains',\n relatedEntity: {\n id: userService.getCurrentUser().tenantId,\n entityType: types.entityType.tenant\n }\n };\n console.log(userService.getCurrentUser());\n vm.relations.push(relation);\n };\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n vm.addRelation();\n saveEntityPromise().then(\n function (entity) {\n $q.all([saveAttributes(entity.id),\n saveRelations(entity.id)]).then(() => {\n updateAliasData();\n $mdDialog.hide();\n });\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n \n function saveRelations(entityId) {\n var tasks = [];\n for (var i=0; i < vm.relations.length; i++) {\n var relation = {\n type: vm.relations[i].relationType\n };\n if (vm.relations[i].direction == types.entitySearchDirection.from) {\n relation.to = vm.relations[i].relatedEntity;\n relation.from = entityId;\n } else {\n relation.to = entityId;\n relation.from = vm.relations[i].relatedEntity;\n }\n tasks.push(entityRelationService.saveRelation(relation));\n }\n return $q.all(tasks);\n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}" } ], "actionCellButton": [ @@ -150,17 +150,7 @@ "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n deviceService = $injector.get('deviceService')\n $rootScope = $injector.get('$rootScope'),\n $q = $injector.get('$q');\n\nopenDeleteEntityDialog();\n\nfunction openDeleteEntityDialog() {\n var title = 'Delete thermostat \"' + entityName + '\"';\n var content = 'Are you sure you want to delete the thermostat \"' +\n entityName + '\"?';\n var confirm = $mdDialog.confirm()\n .targetEvent($event)\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .cancel('Cancel')\n .ok('Delete');\n $mdDialog.show(confirm).then(function() {\n deleteEntity();\n })\n}\n\nfunction deleteEntity() {\n deviceService.deleteDevice(entityId.id).then(\n function success() {\n updateAliasData();\n },\n function fail() {\n showErrorDialog();\n }\n );\n}\n\nfunction updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('entityAliasesChanged', aliasIds);\n });\n}\n\nfunction showErrorDialog() {\n var title = 'Error';\n var content = 'An error occurred while deleting the thermostat. Please try again.';\n var alert = $mdDialog.alert()\n .title(title)\n .htmlContent(content)\n .ariaLabel(title)\n .parent(angular.element($document[0].body))\n .targetEvent($event)\n .multiple(true)\n .clickOutsideToClose(true)\n .ok('CLOSE');\n $mdDialog.show(alert);\n}" } ], - "rowClick": [ - { - "id": "e3928f23-c135-0766-71d5-65ed61e0ce8d", - "name": "show alarm", - "icon": "more_horiz", - "type": "updateDashboardState", - "targetDashboardStateId": "default", - "setEntityId": true, - "stateEntityParamName": "alarm" - } - ] + "rowClick": [] } }, "id": "f33c746c-0dfc-c212-395b-b448c8a17209" @@ -1047,7 +1037,7 @@ }, "states": { "default": { - "name": "Thermostat", + "name": "Thermostats", "root": true, "layouts": { "main": { @@ -1191,10 +1181,12 @@ "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", "alias": "Thermostat-alarm", "filter": { - "type": "stateEntity", + "type": "singleEntity", "resolveMultiple": false, - "stateEntityParamName": "alarm", - "defaultStateEntity": null + "singleEntity": { + "entityType": "CURRENT_TENANT", + "id": null + } } } }, @@ -1233,4 +1225,4 @@ } }, "name": "Thermostats" -} \ No newline at end of file +} From d78193f2e51c11428140f56feecc61e68317efec Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 28 Apr 2020 21:19:53 +0300 Subject: [PATCH 122/196] Test Thermostat devices --- .../data/json/demo/dashboards/theromstats.json | 12 +++++------- .../demo/rule_chains/thermostat_alarms.json | 8 ++++++-- .../DefaultSystemDataLoaderService.java | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/application/src/main/data/json/demo/dashboards/theromstats.json b/application/src/main/data/json/demo/dashboards/theromstats.json index c7ab182171..92d5082031 100644 --- a/application/src/main/data/json/demo/dashboards/theromstats.json +++ b/application/src/main/data/json/demo/dashboards/theromstats.json @@ -1181,12 +1181,10 @@ "id": "ce27a9d0-93bf-b7a4-054d-d0369a8cf813", "alias": "Thermostat-alarm", "filter": { - "type": "singleEntity", - "resolveMultiple": false, - "singleEntity": { - "entityType": "CURRENT_TENANT", - "id": null - } + "type": "entityName", + "resolveMultiple": true, + "entityType": "ASSET", + "entityNameFilter": "Thermostat Alarms" } } }, @@ -1225,4 +1223,4 @@ } }, "name": "Thermostats" -} +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json index 9b7e3401d2..8fd10c88f1 100644 --- a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -24,7 +24,9 @@ "severity": "MAJOR", "propagate": true, "useMessageAlarmData": false, - "relationTypes": [] + "relationTypes": [ + "ToAlarmPropagationAsset" + ] } }, { @@ -54,7 +56,9 @@ "severity": "MINOR", "propagate": true, "useMessageAlarmData": false, - "relationTypes": [] + "relationTypes": [ + "ToAlarmPropagationAsset" + ] } }, { diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index b8d5d26d06..3403cf56e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -36,14 +37,17 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; @@ -81,6 +85,12 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private CustomerService customerService; + @Autowired + private RelationService relationService; + + @Autowired + private AssetService assetService; + @Autowired private DeviceService deviceService; @@ -163,9 +173,17 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { createDevice(demoTenant.getId(), null, DEFAULT_DEVICE_TYPE, "Raspberry Pi Demo Device", "RASPBERRY_PI_DEMO_TOKEN", "Demo device that is used in " + "Raspberry Pi GPIO control sample application"); + Asset thermostatAlarms = new Asset(); + thermostatAlarms.setName("Thermostat Alarms"); + thermostatAlarms.setType("AlarmPropagationAsset"); + thermostatAlarms = assetService.saveAsset(thermostatAlarms); + DeviceId t1Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); DeviceId t2Id = createDevice(demoTenant.getId(), null, "thermostat", "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); + relationService.saveRelation(thermostatAlarms.getTenantId(), new EntityRelation(thermostatAlarms.getId(), t1Id, "ToAlarmPropagationAsset")); + relationService.saveRelation(thermostatAlarms.getTenantId(), new EntityRelation(thermostatAlarms.getId(), t2Id, "ToAlarmPropagationAsset")); + attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), From 8a246cceab0f83a8254ca8a8adcc63703ae60cb6 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Wed, 29 Apr 2020 09:34:33 +0300 Subject: [PATCH 123/196] Clear code (#2691) --- .../demo/dashboards/{theromstats.json => thermostats.json} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename application/src/main/data/json/demo/dashboards/{theromstats.json => thermostats.json} (94%) diff --git a/application/src/main/data/json/demo/dashboards/theromstats.json b/application/src/main/data/json/demo/dashboards/thermostats.json similarity index 94% rename from application/src/main/data/json/demo/dashboards/theromstats.json rename to application/src/main/data/json/demo/dashboards/thermostats.json index 92d5082031..33bf9d6576 100644 --- a/application/src/main/data/json/demo/dashboards/theromstats.json +++ b/application/src/main/data/json/demo/dashboards/thermostats.json @@ -121,7 +121,7 @@ "type": "customPretty", "customHtml": "\n
\n \n
\n

Add thermostat

\n \n \n \n \n
\n \n \n
\n \n \n \n
\n
Thermostat name is required.
\n
\n \n \n High temperature alarm\n \n \n \n \n
\n
High temperature threshold is required.
\n
\n \n \n Low humidity alarm\n \n \n \n \n
\n
Low humidity threshold is required.
\n
\n \n
\n \n \n Create\n Cancel\n \n
\n", "customCss": ".add-entity-form md-input-container {\n padding-right: 10px;\n}\n\n.add-entity-form .boolean-value-input {\n padding-left: 5px;\n}\n\n.add-entity-form .boolean-value-input .checkbox-label {\n margin-bottom: 8px;\n color: rgba(0,0,0,0.54);\n font-size: 12px;\n}\n\n", - "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n types = $injector.get('types'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService'),\n userService = $injector.get('userService'),\n entityRelationService = $injector.get('entityRelationService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n vm.relations = [];\n\n vm.addRelation = function() {\n var relation = {\n direction: types.entitySearchDirection.from,\n relationType: 'Contains',\n relatedEntity: {\n id: userService.getCurrentUser().tenantId,\n entityType: types.entityType.tenant\n }\n };\n console.log(userService.getCurrentUser());\n vm.relations.push(relation);\n };\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n vm.addRelation();\n saveEntityPromise().then(\n function (entity) {\n $q.all([saveAttributes(entity.id),\n saveRelations(entity.id)]).then(() => {\n updateAliasData();\n $mdDialog.hide();\n });\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } \n }\n \n function saveRelations(entityId) {\n var tasks = [];\n for (var i=0; i < vm.relations.length; i++) {\n var relation = {\n type: vm.relations[i].relationType\n };\n if (vm.relations[i].direction == types.entitySearchDirection.from) {\n relation.to = vm.relations[i].relatedEntity;\n relation.from = entityId;\n } else {\n relation.to = entityId;\n relation.from = vm.relations[i].relatedEntity;\n }\n tasks.push(entityRelationService.saveRelation(relation));\n }\n return $q.all(tasks);\n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}" + "customFunction": "var $injector = widgetContext.$scope.$injector;\nvar $mdDialog = $injector.get('$mdDialog'),\n $document = $injector.get('$document'),\n $q = $injector.get('$q'),\n $rootScope = $injector.get('$rootScope'),\n deviceService = $injector.get('deviceService'),\n attributeService = $injector.get('attributeService');\n\nopenAddEntityDialog();\n\nfunction openAddEntityDialog() {\n $mdDialog.show({\n controller: ['$scope','$mdDialog', AddEntityDialogController],\n controllerAs: 'vm',\n template: htmlTemplate,\n locals: {\n entityId: entityId\n },\n parent: angular.element($document[0].body),\n targetEvent: $event,\n multiple: true,\n clickOutsideToClose: false\n });\n}\n\nfunction AddEntityDialogController($scope, $mdDialog) {\n var vm = this;\n vm.attributes = {};\n\n vm.save = function() {\n $scope.addEntityForm.$setPristine();\n saveEntityPromise().then(\n function (entity) {\n saveAttributes(entity.id).then(() => {\n updateAliasData();\n $mdDialog.hide();\n });\n }\n );\n };\n vm.cancel = function() {\n $mdDialog.hide();\n };\n \n \n function saveEntityPromise() {\n var entity = {\n name: vm.entityName,\n type: \"thermostat\"\n };\n return deviceService.saveDevice(entity);\n }\n \n function saveAttributes(entityId) {\n var attributesArray = [];\n for (var key in vm.attributes) {\n attributesArray.push({key: key, value: vm.attributes[key]});\n }\n if (attributesArray.length > 0) {\n return attributeService.saveEntityAttributes(entityId.entityType, entityId.id, \"SERVER_SCOPE\", attributesArray);\n } else {\n return $q.when(null);\n } \n }\n \n function updateAliasData() {\n var aliasIds = [];\n for (var id in widgetContext.aliasController.resolvedAliases) {\n aliasIds.push(id);\n }\n var tasks = [];\n aliasIds.forEach(function(aliasId) {\n widgetContext.aliasController.setAliasUnresolved(aliasId);\n tasks.push(widgetContext.aliasController.getAliasInfo(aliasId));\n });\n $q.all(tasks).then(function() {\n $rootScope.$broadcast('widgetForceReInit');\n });\n }\n}" } ], "actionCellButton": [ @@ -1223,4 +1223,4 @@ } }, "name": "Thermostats" -} \ No newline at end of file +} From ef35ee8c304d743c55c053a0996ce84793daf40e Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Apr 2020 09:41:21 +0300 Subject: [PATCH 124/196] added queue settings to js-executor --- .../config/custom-environment-variables.yml | 12 ++-- msa/js-executor/config/default.yml | 22 +++++-- msa/js-executor/package.json | 2 +- msa/js-executor/queue/awsSqsTemplate.js | 66 ++++++++++++++----- msa/js-executor/queue/kafkaTemplate.js | 57 +++++++++++++++- msa/js-executor/queue/pubSubTemplate.js | 62 +++++++++++++++++ msa/js-executor/queue/rabbitmqTemplate.js | 38 +++++++++-- 7 files changed, 224 insertions(+), 35 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 3beab91b40..88d6341e04 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -17,30 +17,34 @@ service-type: "TB_SERVICE_TYPE" request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" +js: + response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS" + kafka: bootstrap: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" + replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" + topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" pubsub: project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" + queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" aws_sqs: access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" region: "TB_QUEUE_AWS_SQS_REGION" + queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" rabbitmq: - exchange_name: "TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME" host: "TB_QUEUE_RABBIT_MQ_HOST" port: "TB_QUEUE_RABBIT_MQ_PORT" virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" username: "TB_QUEUE_RABBIT_MQ_USERNAME" password: "TB_QUEUE_RABBIT_MQ_PASSWORD" - automatic_recovery_enabled: "TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED" - connection_timeout: "TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT" - handshake_timeout: "TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT" + queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" logger: level: "LOGGER_LEVEL" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index ee38296b31..551aaabdf5 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -15,23 +15,31 @@ # service-type: "kafka" -request_topic: "js.eval.requests" +request_topic: "js_eval.requests" + +js: + response_poll_interval: "25" kafka: bootstrap: # Kafka Bootstrap Servers servers: "localhost:9092" + replication_factor: "1" + topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + +pubsub: + queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" + +aws_sqs: + queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" rabbitmq: - exchange_name: "" host: "localhost" port: "5672" virtual_host: "/" - username: "YOUR_USERNAME" - password: "YOUR_PASSWORD" - automatic_recovery_enabled: "false" - connection_timeout: "60000" - handshake_timeout: "10000" + username: "admin" + password: "password" + queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" logger: diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 6b2ac41a9c..3b496c33e6 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -14,7 +14,7 @@ "dependencies": { "config": "^3.2.2", "js-yaml": "^3.12.0", - "kafkajs": "^1.11.0", + "kafkajs": "^1.12.0", "@google-cloud/pubsub": "^1.7.1", "aws-sdk": "^2.663.0", "amqplib": "^0.5.5", diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index 6bd2f73510..c74341d73d 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -26,10 +26,13 @@ const accessKeyId = config.get('aws_sqs.access_key_id'); const secretAccessKey = config.get('aws_sqs.secret_access_key'); const region = config.get('aws_sqs.region'); const AWS = require('aws-sdk'); +const queueProperties = config.get('aws_sqs.queue-properties'); +const poolInterval = config.get('js.response_poll_interval'); +let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; let sqsClient; -let queueURL; -let responseTopics = new Map(); +let requestQueueURL; +let queueUrls = new Map(); let stopped = false; function AwsSqsProducer() { @@ -41,11 +44,11 @@ function AwsSqsProducer() { headers: headers }); - let responseQueueUrl = responseTopics.get(responseTopic); + let responseQueueUrl = queueUrls.get(topicToSqsQueueName(responseTopic)); if (!responseQueueUrl) { responseQueueUrl = await createQueue(responseTopic); - responseTopics.set(responseTopic, responseQueueUrl); + queueUrls.set(responseTopic, responseQueueUrl); } let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; @@ -69,13 +72,27 @@ function AwsSqsProducer() { sqsClient = new AWS.SQS({apiVersion: '2012-11-05'}); - queueURL = await createQueue(requestTopic); + const queues = await getQueues(); + + queues.forEach(queueUrl => { + const delimiterPosition = queueUrl.lastIndexOf('/'); + const queueName = queueUrl.substring(delimiterPosition + 1); + queueUrls.set(queueName, queueUrl); + }) + + parseQueueProperties(); + + requestQueueURL = queueUrls.get(topicToSqsQueueName(requestTopic)); + if (!requestQueueURL) { + requestQueueURL = await createQueue(requestTopic); + } + const messageProcessor = new JsInvokeMessageProcessor(new AwsSqsProducer()); const params = { MaxNumberOfMessages: 10, - QueueUrl: queueURL, - WaitTimeSeconds: 0.025 + QueueUrl: requestQueueURL, + WaitTimeSeconds: poolInterval / 1000 }; while (!stopped) { const messages = await new Promise((resolve, reject) => { @@ -100,7 +117,7 @@ function AwsSqsProducer() { }); const deleteBatch = { - QueueUrl: queueURL, + QueueUrl: requestQueueURL, Entries: entries }; sqsClient.deleteMessageBatch(deleteBatch, function (err, data) { @@ -120,14 +137,9 @@ function AwsSqsProducer() { })(); function createQueue(topic) { - let queueName = topic.replace(/\./g, '_') + '.fifo'; - let queueParams = { - QueueName: queueName, Attributes: { - FifoQueue: 'true', - ContentBasedDeduplication: 'true' + let queueName = topicToSqsQueueName(topic); + let queueParams = {QueueName: queueName, Attributes: queueAttributes}; - } - }; return new Promise((resolve, reject) => { sqsClient.createQueue(queueParams, function (err, data) { if (err) { @@ -139,6 +151,30 @@ function createQueue(topic) { }); } +function getQueues() { + return new Promise((resolve, reject) => { + sqsClient.listQueues(function (err, data) { + if (err) { + reject(err); + } else { + resolve(data.QueueUrls); + } + }); + }); +} + +function topicToSqsQueueName(topic) { + return topic.replace(/\./g, '_') + '.fifo'; +} + +function parseQueueProperties() { + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); +} + process.on('exit', () => { stopped = true; logger.info('Aws Sqs client stopped.'); diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 38e713a7db..f0fde2952c 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -19,13 +19,28 @@ const config = require('config'), JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), logger = require('../config/logger')._logger('kafkaTemplate'), KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; +const replicationFactor = config.get('kafka.replication_factor'); +const topicProperties = config.get('kafka.topic-properties'); let kafkaClient; +let kafkaAdmin; let consumer; let producer; +const topics = []; +const configEntries = []; + function KafkaProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { + + if (!topics.includes(responseTopic)) { + let createResponseTopicResult = await createTopic(responseTopic); + topics.push(responseTopic); + if (createResponseTopicResult) { + logger.info('Created new topic: %s', requestTopic); + } + } + let headersData = headers.data; headersData = Object.fromEntries(Object.entries(headersData).map(([key, value]) => [key, Buffer.from(value)])); return producer.send( @@ -47,10 +62,10 @@ function KafkaProducer() { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); - const kafkaRequestTopic = config.get('request_topic'); + const requestTopic = config.get('request_topic'); logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); - logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); + logger.info('Kafka Requests Topic: %s', requestTopic); kafkaClient = new Kafka({ brokers: kafkaBootstrapServers.split(','), @@ -58,12 +73,23 @@ function KafkaProducer() { logCreator: KafkaJsWinstonLogCreator }); + parseTopicProperties(); + + kafkaAdmin = kafkaClient.admin(); + await kafkaAdmin.connect(); + + let createRequestTopicResult = await createTopic(requestTopic); + + if (createRequestTopicResult) { + logger.info('Created new topic: %s', requestTopic); + } + consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); producer = kafkaClient.producer(); const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); await consumer.connect(); await producer.connect(); - await consumer.subscribe({topic: kafkaRequestTopic}); + await consumer.subscribe({topic: requestTopic}); logger.info('Started ThingsBoard JavaScript Executor Microservice.'); await consumer.run({ @@ -90,12 +116,37 @@ function KafkaProducer() { } })(); +function createTopic(topic) { + return kafkaAdmin.createTopics({ + topics: [{ + topic: topic, + replicationFactor: replicationFactor, + configEntries: configEntries + }] + }); +} + +function parseTopicProperties() { + const props = topicProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + configEntries.push({name: p.substring(0, delimiterPosition), value: p.substring(delimiterPosition + 1)}); + }); +} + process.on('exit', () => { exit(0); }); async function exit(status) { logger.info('Exiting with status: %d ...', status); + + if (kafkaAdmin) { + logger.info('Stopping Kafka Admin...'); + await kafkaAdmin.disconnect(); + logger.info('Kafka Admin stopped.'); + } + if (consumer) { logger.info('Stopping Kafka Consumer...'); let _consumer = consumer; diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index c8b39f7d6b..708c1d56a4 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -24,11 +24,21 @@ const {PubSub} = require('@google-cloud/pubsub'); const projectId = config.get('pubsub.project_id'); const credentials = JSON.parse(config.get('pubsub.service_account')); const requestTopic = config.get('request_topic'); +const queueProperties = config.get('pubsub.queue-properties'); let pubSubClient; +const topics = []; +const subscriptions = []; +let queueProps = []; + function PubSubProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { + + if (!(subscriptions.includes(responseTopic) && topics.includes(requestTopic))) { + await createTopic(requestTopic); + } + let data = JSON.stringify( { key: scriptId, @@ -45,6 +55,28 @@ function PubSubProducer() { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); pubSubClient = new PubSub({projectId: projectId, credentials: credentials}); + parseQueueProperties(); + + const topicList = await pubSubClient.getTopics(); + + if (topicList) { + topicList[0].forEach(topic => { + topics.push(getName(topic.name)); + }); + } + + const subscriptionList = await pubSubClient.getSubscriptions(); + + if (subscriptionList) { + topicList[0].forEach(sub => { + subscriptions.push(getName(sub.name)); + }); + } + + if (!(subscriptions.includes(requestTopic) && topics.includes(requestTopic))) { + await createTopic(requestTopic); + } + const subscription = pubSubClient.subscription(requestTopic); const messageProcessor = new JsInvokeMessageProcessor(new PubSubProducer()); @@ -64,6 +96,36 @@ function PubSubProducer() { } })(); +async function createTopic(topic) { + if (!topics.includes(topic)) { + await pubSubClient.createTopic(topic); + topics.push(topic); + logger.info('Created new Pub/Sub topic: %s', topic); + } + await createSubscription(topic) +} + +async function createSubscription(topic) { + if (!subscriptions.includes(topic)) { + await pubSubClient.topic(topic).createSubscription(topic); + subscriptions.push(topic); + logger.info('Created new Pub/Sub subscription: %s', topic); + } +} + +function parseQueueProperties() { + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); +} + +function getName(fullName) { + const delimiterPosition = fullName.lastIndexOf('/'); + return fullName.substring(delimiterPosition + 1); +} + process.on('exit', () => { exit(0); }); diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js index 0b48cdd62f..e33409fc5a 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.js +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -21,7 +21,17 @@ const config = require('config'), logger = require('../config/logger')._logger('rabbitmqTemplate'); const requestTopic = config.get('request_topic'); +const host = config.get('rabbitmq.host'); +const port = config.get('rabbitmq.port'); +const vhost = config.get('rabbitmq.virtual_host'); +const username = config.get('rabbitmq.username'); +const password = config.get('rabbitmq.password'); +const queueProperties = config.get('rabbitmq.queue-properties'); +const poolInterval = config.get('js.response_poll_interval'); + const amqp = require('amqplib/callback_api'); + +let queueParams = {durable: false, exclusive: false, autoDelete: false}; let connection; let channel; let stopped = false; @@ -58,10 +68,11 @@ function RabbitMqProducer() { (async () => { try { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + const url = `amqp://${host}:${port}${vhost}`; - amqp.credentials.amqplain('admin', 'password'); + amqp.credentials.amqplain(username, password); connection = await new Promise((resolve, reject) => { - amqp.connect('amqp://localhost:5672/', function (err, connection) { + amqp.connect(url, function (err, connection) { if (err) { reject(err); } else { @@ -80,6 +91,8 @@ function RabbitMqProducer() { }); }); + parseQueueProperties(); + await createQueue(requestTopic); const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer()); @@ -98,6 +111,8 @@ function RabbitMqProducer() { if (message) { messageProcessor.onJsInvokeMessage(message.content.toString('utf8')); channel.ack(message); + } else { + await sleep(poolInterval); } } } catch (e) { @@ -107,10 +122,17 @@ function RabbitMqProducer() { } })(); +function parseQueueProperties() { + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + queueParams[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); +} + function createQueue(topic) { - let params = {durable: false}; return new Promise((resolve, reject) => { - channel.assertQueue(topic, params, function (err, data) { + channel.assertQueue(topic, queueParams, function (err, data) { if (err) { reject(err); } else { @@ -120,6 +142,12 @@ function createQueue(topic) { }); } +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + process.on('exit', () => { exit(0); }); @@ -146,4 +174,4 @@ async function exit(status) { } else { process.exit(status); } -} \ No newline at end of file +} From 79b4411c69f1c49233ecf21322b3ece695d8ecc5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Apr 2020 09:45:24 +0300 Subject: [PATCH 125/196] Synchronization of device connect processing in gateways --- .../mqtt/session/GatewaySessionHandler.java | 74 ++++++++++++------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 65393d6939..5dae247f1c 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -69,7 +69,8 @@ public class GatewaySessionHandler { private final TransportService transportService; private final DeviceInfoProto gateway; private final UUID sessionId; - private final Map devices; + private final ConcurrentMap devices; + private final ConcurrentMap> deviceFutures; private final ConcurrentMap mqttQoSMap; private final ChannelHandlerContext channel; private final DeviceSessionCtx deviceSessionCtx; @@ -81,6 +82,7 @@ public class GatewaySessionHandler { this.gateway = deviceSessionCtx.getDeviceInfo(); this.sessionId = sessionId; this.devices = new ConcurrentHashMap<>(); + this.deviceFutures = new ConcurrentHashMap<>(); this.mqttQoSMap = deviceSessionCtx.getMqttQoSMap(); this.channel = deviceSessionCtx.getChannel(); } @@ -106,35 +108,51 @@ public class GatewaySessionHandler { } private ListenableFuture onDeviceConnect(String deviceName, String deviceType) { - SettableFuture future = SettableFuture.create(); + SettableFuture future; GatewayDeviceSessionCtx result = devices.get(deviceName); if (result == null) { - transportService.process(GetOrCreateDeviceFromGatewayRequestMsg.newBuilder() - .setDeviceName(deviceName) - .setDeviceType(deviceType) - .setGatewayIdMSB(gateway.getDeviceIdMSB()) - .setGatewayIdLSB(gateway.getDeviceIdLSB()).build(), - new TransportServiceCallback() { - @Override - public void onSuccess(GetOrCreateDeviceFromGatewayResponseMsg msg) { - GatewayDeviceSessionCtx deviceSessionCtx = new GatewayDeviceSessionCtx(GatewaySessionHandler.this, msg.getDeviceInfo(), mqttQoSMap); - if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { - SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); - transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); - transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); - transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); - } - future.set(devices.get(deviceName)); - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to process device connect command: {}", sessionId, deviceName, e); - future.setException(e); - } - }); + synchronized (deviceFutures) { + future = deviceFutures.get(deviceName); + if (future == null) { + final SettableFuture futureToSet = SettableFuture.create(); + deviceFutures.put(deviceName, futureToSet); + future = futureToSet; + try { + transportService.process(GetOrCreateDeviceFromGatewayRequestMsg.newBuilder() + .setDeviceName(deviceName) + .setDeviceType(deviceType) + .setGatewayIdMSB(gateway.getDeviceIdMSB()) + .setGatewayIdLSB(gateway.getDeviceIdLSB()).build(), + new TransportServiceCallback() { + @Override + public void onSuccess(GetOrCreateDeviceFromGatewayResponseMsg msg) { + GatewayDeviceSessionCtx deviceSessionCtx = new GatewayDeviceSessionCtx(GatewaySessionHandler.this, msg.getDeviceInfo(), mqttQoSMap); + if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { + SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); + transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); + transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); + transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); + } + futureToSet.set(devices.get(deviceName)); + deviceFutures.remove(deviceName); + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to process device connect command: {}", sessionId, deviceName, e); + futureToSet.setException(e); + deviceFutures.remove(deviceName); + } + }); + } catch (Throwable e) { + deviceFutures.remove(deviceName); + throw e; + } + } + } } else { + future = SettableFuture.create(); future.set(result); } return future; From fa5c051476e36982a8fd1e307555eaf03760e21d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 29 Apr 2020 09:50:22 +0300 Subject: [PATCH 126/196] Fix demo data loader: set tenantId for termostatAlarms asset --- .../server/service/install/DefaultSystemDataLoaderService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 3403cf56e2..329dc37d3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -174,6 +174,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { "Raspberry Pi GPIO control sample application"); Asset thermostatAlarms = new Asset(); + thermostatAlarms.setTenantId(demoTenant.getId()); thermostatAlarms.setName("Thermostat Alarms"); thermostatAlarms.setType("AlarmPropagationAsset"); thermostatAlarms = assetService.saveAsset(thermostatAlarms); From a38ee80e0957d52bc21f0a201b081edf4d3420f2 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Apr 2020 10:02:00 +0300 Subject: [PATCH 127/196] Refactoring of device creation using gateway --- .../mqtt/session/GatewaySessionHandler.java | 104 +++++++++++------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 5dae247f1c..caa434e4da 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -54,6 +54,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; /** * Created by ashvayka on 19.01.17. @@ -69,6 +71,7 @@ public class GatewaySessionHandler { private final TransportService transportService; private final DeviceInfoProto gateway; private final UUID sessionId; + private final Lock deviceCreationLock = new ReentrantLock(); private final ConcurrentMap devices; private final ConcurrentMap> deviceFutures; private final ConcurrentMap mqttQoSMap; @@ -108,53 +111,70 @@ public class GatewaySessionHandler { } private ListenableFuture onDeviceConnect(String deviceName, String deviceType) { - SettableFuture future; GatewayDeviceSessionCtx result = devices.get(deviceName); if (result == null) { - synchronized (deviceFutures) { - future = deviceFutures.get(deviceName); - if (future == null) { - final SettableFuture futureToSet = SettableFuture.create(); - deviceFutures.put(deviceName, futureToSet); - future = futureToSet; - try { - transportService.process(GetOrCreateDeviceFromGatewayRequestMsg.newBuilder() - .setDeviceName(deviceName) - .setDeviceType(deviceType) - .setGatewayIdMSB(gateway.getDeviceIdMSB()) - .setGatewayIdLSB(gateway.getDeviceIdLSB()).build(), - new TransportServiceCallback() { - @Override - public void onSuccess(GetOrCreateDeviceFromGatewayResponseMsg msg) { - GatewayDeviceSessionCtx deviceSessionCtx = new GatewayDeviceSessionCtx(GatewaySessionHandler.this, msg.getDeviceInfo(), mqttQoSMap); - if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { - SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); - transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); - transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); - transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); - transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); - } - futureToSet.set(devices.get(deviceName)); - deviceFutures.remove(deviceName); - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to process device connect command: {}", sessionId, deviceName, e); - futureToSet.setException(e); - deviceFutures.remove(deviceName); - } - }); - } catch (Throwable e) { - deviceFutures.remove(deviceName); - throw e; - } + deviceCreationLock.lock(); + try { + result = devices.get(deviceName); + if (result == null) { + return getDeviceCreationFuture(deviceName, deviceType); + } else { + return toCompletedFuture(result); } + } finally { + deviceCreationLock.unlock(); } } else { - future = SettableFuture.create(); - future.set(result); + return toCompletedFuture(result); } + } + + private ListenableFuture getDeviceCreationFuture(String deviceName, String deviceType) { + SettableFuture future = deviceFutures.get(deviceName); + if (future == null) { + final SettableFuture futureToSet = SettableFuture.create(); + deviceFutures.put(deviceName, futureToSet); + try { + transportService.process(GetOrCreateDeviceFromGatewayRequestMsg.newBuilder() + .setDeviceName(deviceName) + .setDeviceType(deviceType) + .setGatewayIdMSB(gateway.getDeviceIdMSB()) + .setGatewayIdLSB(gateway.getDeviceIdLSB()).build(), + new TransportServiceCallback() { + @Override + public void onSuccess(GetOrCreateDeviceFromGatewayResponseMsg msg) { + GatewayDeviceSessionCtx deviceSessionCtx = new GatewayDeviceSessionCtx(GatewaySessionHandler.this, msg.getDeviceInfo(), mqttQoSMap); + if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) { + SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo(); + transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx); + transportService.process(deviceSessionInfo, DefaultTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null); + transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null); + transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null); + } + futureToSet.set(devices.get(deviceName)); + deviceFutures.remove(deviceName); + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to process device connect command: {}", sessionId, deviceName, e); + futureToSet.setException(e); + deviceFutures.remove(deviceName); + } + }); + return futureToSet; + } catch (Throwable e) { + deviceFutures.remove(deviceName); + throw e; + } + } else { + return future; + } + } + + private ListenableFuture toCompletedFuture(GatewayDeviceSessionCtx result) { + SettableFuture future = SettableFuture.create(); + future.set(result); return future; } From cce6a44adc8a701ec0c08b7bfe91ff6f5388c138 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Apr 2020 10:14:05 +0300 Subject: [PATCH 128/196] refactored queue factories --- .../provider/AwsSqsMonolithQueueFactory.java | 5 +-- .../provider/AwsSqsTbCoreQueueFactory.java | 28 +++++++++++++- .../AwsSqsTbRuleEngineQueueFactory.java | 30 ++++++++++++++- .../provider/PubSubTbCoreQueueFactory.java | 38 ++++++++++++++++--- .../PubSubTbRuleEngineQueueFactory.java | 28 +++++++++++++- .../provider/RabbitMqTbCoreQueueFactory.java | 28 +++++++++++++- .../RabbitMqTbRuleEngineQueueFactory.java | 28 +++++++++++++- 7 files changed, 170 insertions(+), 15 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index c86baf9c48..c1d724c207 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -20,7 +20,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.*; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; @@ -31,8 +32,6 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index a7349091bd..4889a7182d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,11 +32,13 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -44,6 +48,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") @@ -55,6 +60,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -68,6 +74,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbAwsSqsQueueAttributes sqsQueueAttributes) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; @@ -75,6 +82,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -133,8 +141,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 3056eced2c..cf43bc5fe4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -27,11 +29,13 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -41,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") @@ -51,6 +56,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -61,12 +67,14 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { + TbAwsSqsQueueAttributes sqsQueueAttributes, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.sqsSettings = sqsSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -113,8 +121,26 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 0a23d58e46..7d85b5c3ba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,21 +30,24 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") @@ -53,6 +58,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin jsExecutorAdmin; @@ -64,12 +70,14 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueTransportApiSettings transportApiSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); @@ -127,8 +135,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 79501130ed..a6105fea0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,10 +41,12 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") @@ -52,6 +57,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbQueueRuleEngineSettings ruleEngineSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -63,12 +69,14 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -116,8 +124,26 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index d84bbedbd2..65d582c4bc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,6 +32,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -40,10 +43,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") @@ -55,6 +60,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -68,6 +74,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments) { this.rabbitMqSettings = rabbitMqSettings; this.coreSettings = coreSettings; @@ -75,6 +82,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); @@ -133,8 +141,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index a18f427fab..26abf78164 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,10 +41,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") @@ -52,6 +57,7 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -62,12 +68,14 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbRabbitMqSettings rabbitMqSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.rabbitMqSettings = rabbitMqSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); @@ -114,8 +122,26 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy From 8c413f8b9c0b5ba530e81cf4ddf02995c136fcc8 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Apr 2020 10:42:18 +0300 Subject: [PATCH 129/196] added queue params to pubsub js-executor --- msa/js-executor/queue/awsSqsTemplate.js | 2 +- msa/js-executor/queue/pubSubTemplate.js | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index c74341d73d..297509ca9b 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -32,7 +32,7 @@ const poolInterval = config.get('js.response_poll_interval'); let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; let sqsClient; let requestQueueURL; -let queueUrls = new Map(); +const queueUrls = new Map(); let stopped = false; function AwsSqsProducer() { diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 708c1d56a4..0db0d19f05 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -30,7 +30,7 @@ let pubSubClient; const topics = []; const subscriptions = []; -let queueProps = []; +const queueProps = []; function PubSubProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { @@ -107,7 +107,12 @@ async function createTopic(topic) { async function createSubscription(topic) { if (!subscriptions.includes(topic)) { - await pubSubClient.topic(topic).createSubscription(topic); + await pubSubClient.createSubscription(topic, topic, { + topic: topic, + subscription: topic, + ackDeadlineSeconds: queueProps['ackDeadlineInSec'], + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} + }); subscriptions.push(topic); logger.info('Created new Pub/Sub subscription: %s', topic); } From 5fa2e2fc49a7fb6f62da9db8aafa208c0f2edee8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Apr 2020 11:50:02 +0300 Subject: [PATCH 130/196] Added automatic creation of relations for demo dashboard --- .../json/demo/dashboards/thermostats.json | 13 +-- .../demo/rule_chains/root_rule_chain.json | 86 ++++++++++++++----- 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/application/src/main/data/json/demo/dashboards/thermostats.json b/application/src/main/data/json/demo/dashboards/thermostats.json index 33bf9d6576..f91d987447 100644 --- a/application/src/main/data/json/demo/dashboards/thermostats.json +++ b/application/src/main/data/json/demo/dashboards/thermostats.json @@ -985,8 +985,8 @@ "showTooltip": true, "autocloseTooltip": true, "defaultCenterPosition": [ - 0, - 0 + 37.7749, + -122.4194 ], "customProviderTileUrl": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", "showTooltipAction": "click", @@ -999,10 +999,11 @@ "animate": true, "maxClusterRadius": 80, "removeOutsideVisibleBounds": true, - "defaultZoomLevel": 12, + "defaultZoomLevel": 5, "labelFunction": "var color;\nif(dsData[dsIndex].active !== \"true\"){\n color = 'rgb(255, 0, 0)';\n} else {\n color = 'rgb(39, 134, 34)';\n}\nreturn '' + \n '${entityLabel}' + \n ''", "markerImageFunction": "var res;\nif(dsData[dsIndex].active !== \"true\"){\n\tvar res = {\n\t url: images[0],\n\t size: 48\n\t}\n} else {\n var res = {\n\t url: images[1],\n\t size: 48\n\t}\n}\nreturn res;", - "useLabelFunction": true + "useLabelFunction": true, + "useDefaultCenterPosition": true }, "title": "New Markers Placement - OpenStreetMap", "dropShadow": true, @@ -1182,7 +1183,7 @@ "alias": "Thermostat-alarm", "filter": { "type": "entityName", - "resolveMultiple": true, + "resolveMultiple": false, "entityType": "ASSET", "entityNameFilter": "Thermostat Alarms" } @@ -1223,4 +1224,4 @@ } }, "name": "Thermostats" -} +} \ No newline at end of file diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index b68d070861..865d8cb8e6 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -3,13 +3,25 @@ "additionalInfo": null, "name": "Root Rule Chain", "firstRuleNodeId": null, - "root": true, + "root": false, "debugMode": false, "configuration": null }, "metadata": { - "firstNodeIndex": 2, + "firstNodeIndex": 3, "nodes": [ + { + "additionalInfo": { + "layoutX": 1069, + "layoutY": 267 + }, + "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", + "name": "Is Thermostat?", + "debugMode": true, + "configuration": { + "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";" + } + }, { "additionalInfo": { "layoutX": 824, @@ -48,8 +60,8 @@ }, { "additionalInfo": { - "layoutX": 825, - "layoutY": 266 + "layoutX": 839, + "layoutY": 345 }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log RPC from Device", @@ -60,8 +72,8 @@ }, { "additionalInfo": { - "layoutX": 825, - "layoutY": 379 + "layoutX": 832, + "layoutY": 407 }, "type": "org.thingsboard.rule.engine.action.TbLogNode", "name": "Log Other", @@ -93,51 +105,81 @@ "configuration": { "jsScript": "return metadata[\"deviceType\"] === \"thermostat\";" } + }, + { + "additionalInfo": { + "layoutX": 1090, + "layoutY": 360 + }, + "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode", + "name": "Relate to Asset", + "debugMode": true, + "configuration": { + "direction": "FROM", + "relationType": "ToAlarmPropagationAsset", + "entityType": "ASSET", + "entityNamePattern": "Thermostat Alarms", + "entityTypePattern": "AlarmPropagationAsset", + "entityCacheExpiration": 300, + "createEntityIfNotExists": true, + "changeOriginatorToRelatedEntity": false, + "removeCurrentRelations": false + } } ], "connections": [ { "fromIndex": 0, - "toIndex": 6, + "toIndex": 8, + "type": "True" + }, + { + "fromIndex": 1, + "toIndex": 7, "type": "Success" }, { - "fromIndex": 2, - "toIndex": 4, + "fromIndex": 3, + "toIndex": 5, "type": "Other" }, { - "fromIndex": 2, - "toIndex": 1, + "fromIndex": 3, + "toIndex": 2, "type": "Post attributes" }, { - "fromIndex": 2, - "toIndex": 0, + "fromIndex": 3, + "toIndex": 1, "type": "Post telemetry" }, { - "fromIndex": 2, - "toIndex": 3, + "fromIndex": 3, + "toIndex": 4, "type": "RPC Request from Device" }, { - "fromIndex": 2, - "toIndex": 5, + "fromIndex": 3, + "toIndex": 6, "type": "RPC Request to Device" + }, + { + "fromIndex": 3, + "toIndex": 0, + "type": "Entity Created" } ], "ruleChainConnections": [ { - "fromIndex": 6, + "fromIndex": 7, "targetRuleChainId": { "entityType": "RULE_CHAIN", - "id": "83d42540-85fd-11ea-aee2-794850541ced" + "id": "25e26570-89ed-11ea-a650-cd6e14e633bd" }, "additionalInfo": { - "layoutX": 1088, - "layoutY": 203, - "ruleChainNodeId": "rule-chain-node-9" + "layoutX": 1109, + "layoutY": 182, + "ruleChainNodeId": "rule-chain-node-10" }, "type": "True" } From 6b6d4702a8f5100560b919000d0665e8462bcf9d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 29 Apr 2020 14:30:34 +0300 Subject: [PATCH 131/196] Update package.json versions --- msa/js-executor/package.json | 2 +- msa/web-ui/package.json | 2 +- ui/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index c87b62c6eb..0cdafffef1 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-js-executor", "private": true, - "version": "2.4.3", + "version": "2.5.0", "description": "ThingsBoard JavaScript Executor Microservice", "main": "server.js", "bin": "server.js", diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 188084d375..176791d65a 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-web-ui", "private": true, - "version": "2.4.3", + "version": "2.5.0", "description": "ThingsBoard Web UI Microservice", "main": "server.js", "bin": "server.js", diff --git a/ui/package.json b/ui/package.json index 96c94bf677..6181e70de0 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard", "private": true, - "version": "2.4.3", + "version": "2.5.0", "description": "ThingsBoard UI", "licenses": [ { From 43b2eedbd5cc82caa7e5c66ca7125de7d4702b81 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Apr 2020 18:27:53 +0300 Subject: [PATCH 132/196] azure service bus js-executor --- .../azure/servicebus/TbServiceBusAdmin.java | 8 +- .../ServiceBusMonolithQueueFactory.java | 30 ++- .../ServiceBusTbCoreQueueFactory.java | 28 ++- .../ServiceBusTbRuleEngineQueueFactory.java | 28 ++- .../api/jsInvokeMessageProcessor.js | 3 +- .../config/custom-environment-variables.yml | 9 +- msa/js-executor/config/default.yml | 2 + msa/js-executor/package.json | 2 + msa/js-executor/queue/awsSqsTemplate.js | 4 +- msa/js-executor/queue/kafkaTemplate.js | 2 +- msa/js-executor/queue/pubSubTemplate.js | 4 +- msa/js-executor/queue/rabbitmqTemplate.js | 8 +- msa/js-executor/queue/serviceBusTemplate.js | 194 ++++++++++++++++++ msa/js-executor/server.js | 13 +- .../rule/engine/delay/TbMsgDelayNode.java | 2 +- 15 files changed, 314 insertions(+), 23 deletions(-) create mode 100644 msa/js-executor/queue/serviceBusTemplate.js diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index 229d6b4244..f108de4a43 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.azure.servicebus; import com.microsoft.azure.servicebus.management.ManagementClient; import com.microsoft.azure.servicebus.management.QueueDescription; import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsException; import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; @@ -71,7 +72,12 @@ public class TbServiceBusAdmin implements TbQueueAdmin { client.createQueue(queueDescription); queues.add(topic); } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to create queue: [{}]", topic, e); + if (e instanceof MessagingEntityAlreadyExistsException) { + queues.add(topic); + log.info("[{}] queue already exists.", topic); + } else { + log.error("Failed to create queue: [{}]", topic, e); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 3c82e18d91..3db6496b7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -35,19 +37,20 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") @@ -60,6 +63,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -73,6 +77,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; @@ -81,6 +86,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -152,8 +158,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index bcb48630f9..a3ba577f06 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -34,15 +36,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") @@ -54,6 +59,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -67,6 +73,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; @@ -74,6 +81,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -132,8 +140,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 1f492596a1..e4f6dbfbb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -32,15 +34,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") @@ -51,6 +56,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -61,12 +67,14 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbServiceBusSettings serviceBusSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.serviceBusSettings = serviceBusSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -113,8 +121,26 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index f6da6886e7..79ea505bcf 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -38,12 +38,11 @@ function JsInvokeMessageProcessor(producer) { this.executedScriptsCounter = 0; } -JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(messageStr) { +JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { let requestId; let responseTopic; try { - let message = JSON.parse(messageStr); let request = JSON.parse(Buffer.from(message.data).toString('utf8')); let headers = message.headers; let buf = Buffer.from(headers.data['requestId']); diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 88d6341e04..b290719739 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "TB_SERVICE_TYPE" +service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" js: @@ -46,6 +46,13 @@ rabbitmq: password: "TB_QUEUE_RABBIT_MQ_PASSWORD" queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" +service_bus: + namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" + sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" + sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" + max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" + queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" + logger: level: "LOGGER_LEVEL" path: "LOG_FOLDER" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 551aaabdf5..3155b051dc 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -41,6 +41,8 @@ rabbitmq: password: "password" queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" +service_bus: + queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" logger: level: "info" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 3b496c33e6..695b6e7faf 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -18,6 +18,8 @@ "@google-cloud/pubsub": "^1.7.1", "aws-sdk": "^2.663.0", "amqplib": "^0.5.5", + "@azure/service-bus": "^1.1.6", + "azure-sb": "^0.11.1", "long": "^4.0.0", "uuid-parse": "^1.0.0", "winston": "^3.0.0", diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index 297509ca9b..0396824af1 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -113,7 +113,7 @@ function AwsSqsProducer() { Id: message.MessageId, ReceiptHandle: message.ReceiptHandle }); - messageProcessor.onJsInvokeMessage(message.Body); + messageProcessor.onJsInvokeMessage(JSON.parse(message.Body)); }); const deleteBatch = { @@ -187,7 +187,7 @@ async function exit(status) { logger.info('Stopping Aws Sqs client.') try { await sqsClient.close(); - logger.info('Aws Sqs client is stopped.') + logger.info('Aws Sqs client stopped.') process.exit(status); } catch (e) { logger.info('Aws Sqs client stop error.'); diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index f0fde2952c..699ca7be00 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -105,7 +105,7 @@ function KafkaProducer() { msg.key = key.toString('utf8'); msg.data = [...data]; msg.headers = {data: headers} - messageProcessor.onJsInvokeMessage(JSON.stringify(msg)); + messageProcessor.onJsInvokeMessage(msg); }, }); diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 0db0d19f05..17e1b56e1d 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -83,7 +83,7 @@ function PubSubProducer() { const messageHandler = message => { - messageProcessor.onJsInvokeMessage(message.data.toString('utf8')); + messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8'))); message.ack(); }; @@ -141,7 +141,7 @@ async function exit(status) { logger.info('Stopping Pub/Sub client.') try { await pubSubClient.close(); - logger.info('Pub/Sub client is stopped.') + logger.info('Pub/Sub client stopped.') process.exit(status); } catch (e) { logger.info('Pub/Sub client stop error.'); diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js index e33409fc5a..1a2905c3a0 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.js +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -109,7 +109,7 @@ function RabbitMqProducer() { }); if (message) { - messageProcessor.onJsInvokeMessage(message.content.toString('utf8')); + messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); channel.ack(message); } else { await sleep(poolInterval); @@ -132,7 +132,7 @@ function parseQueueProperties() { function createQueue(topic) { return new Promise((resolve, reject) => { - channel.assertQueue(topic, queueParams, function (err, data) { + channel.assertQueue(topic, queueParams, function (err) { if (err) { reject(err); } else { @@ -158,14 +158,14 @@ async function exit(status) { if (channel) { logger.info('Stopping RabbitMq chanel.') await channel.close(); - logger.info('RabbitMq chanel is stopped'); + logger.info('RabbitMq chanel stopped'); } if (connection) { logger.info('Stopping RabbitMq connection.') try { await connection.close(); - logger.info('RabbitMq client is connection.') + logger.info('RabbitMq client connection.') process.exit(status); } catch (e) { logger.info('RabbitMq connection stop error.'); diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js new file mode 100644 index 0000000000..034921afb7 --- /dev/null +++ b/msa/js-executor/queue/serviceBusTemplate.js @@ -0,0 +1,194 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('serviceBusTemplate'); +const {ServiceBusClient, ReceiveMode} = require("@azure/service-bus"); +const azure = require('azure-sb'); + +const requestTopic = config.get('request_topic'); +const namespaceName = config.get('service_bus.namespace_name'); +const sasKeyName = config.get('service_bus.sas_key_name'); +const sasKey = config.get('service_bus.sas_key'); +const queueProperties = config.get('service_bus.queue-properties'); + +let sbClient; +let receiverClient; +let receiver; +let serviceBusService; + +let queueOptions = {}; +const queues = []; +const senderMap = new Map(); + +function ServiceBusProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + if (!queues.includes(requestTopic)) { + await createQueueIfNotExist(requestTopic); + queues.push(requestTopic); + } + + let customSender = senderMap.get(responseTopic); + + if (!customSender) { + customSender = new CustomSender(responseTopic); + senderMap.set(responseTopic, customSender); + } + + let data = { + key: scriptId, + data: [...rawResponse], + headers: headers + }; + + return customSender.send({body: data}); + } +} + +function CustomSender(topic) { + this.queueClient = sbClient.createQueueClient(topic); + this.sender = this.queueClient.createSender(); + + this.send = async (message) => { + return this.sender.send(message); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + + const connectionString = `Endpoint=sb://${namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${sasKeyName};SharedAccessKey=${sasKey}`; + sbClient = ServiceBusClient.createFromConnectionString(connectionString); + serviceBusService = azure.createServiceBusService(connectionString); + + parseQueueProperties(); + + await new Promise((resolve, reject) => { + serviceBusService.listQueues((err, data) => { + if (err) { + reject(err); + } else { + data.forEach(queue => { + queues.push(queue.QueueName); + }); + resolve(); + } + }); + }); + + if (!queues.includes(requestTopic)) { + await createQueueIfNotExist(requestTopic); + queues.push(requestTopic); + } + + receiverClient = sbClient.createQueueClient(requestTopic); + receiver = receiverClient.createReceiver(ReceiveMode.peekLock); + + const messageProcessor = new JsInvokeMessageProcessor(new ServiceBusProducer()); + + const messageHandler = async (message) => { + if (message) { + messageProcessor.onJsInvokeMessage(message.body); + await message.complete(); + } + }; + const errorHandler = (error) => { + logger.error('Failed to receive message from queue.', error); + }; + receiver.registerMessageHandler(messageHandler, errorHandler); + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +async function createQueueIfNotExist(topic) { + return new Promise((resolve, reject) => { + serviceBusService.createQueueIfNotExists(topic, queueOptions, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +} + +function parseQueueProperties() { + let properties = {}; + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); + queueOptions = { + MaxSizeInMegabytes: properties['maxSizeInMb'], + DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, + LockDuration: `PT${properties['lockDurationInSec']}S` + }; +} + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + logger.info('Stopping Azure Service Bus resources...') + if (receiver) { + try { + await receiver.close(); + } catch (e) { + + } + } + + if (receiverClient) { + try { + await receiverClient.close(); + } catch (e) { + + } + } + + senderMap.forEach((k, v) => { + try { + v.sender.close(); + } catch (e) { + + } + try { + v.queueClient.close(); + } catch (e) { + + } + }); + + if (sbClient) { + try { + sbClient.close(); + } catch (e) { + + } + } + logger.info('Azure Service Bus resources stopped.') + process.exit(status); +} \ No newline at end of file diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index bd84289200..58361016c4 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -21,22 +21,27 @@ switch (serviceType) { case 'kafka': logger.info('Starting kafka template.'); require('./queue/kafkaTemplate'); - logger.info('kafka template is started.'); + logger.info('kafka template started.'); break; case 'pubsub': logger.info('Starting Pub/Sub template.') require('./queue/pubSubTemplate'); - logger.info('Pub/Sub template is started.') + logger.info('Pub/Sub template started.') break; case 'aws-sqs': logger.info('Starting Aws Sqs template.') require('./queue/awsSqsTemplate'); - logger.info('Aws Sqs template is started.') + logger.info('Aws Sqs template started.') break; case 'rabbitmq': logger.info('Starting RabbitMq template.') require('./queue/rabbitmqTemplate'); - logger.info('RabbitMq template is started.') + logger.info('RabbitMq template started.') + break; + case 'service-bus': + logger.info('Starting Azure Service Bus template.') + require('./queue/serviceBusTemplate'); + logger.info('Azure Service Bus template started.') break; default: logger.error('Unknown service type: ', serviceType); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 1807c5c4fe..e2bb07fb6f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -41,7 +41,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; name = "delay", configClazz = TbMsgDelayNodeConfiguration.class, nodeDescription = "Delays incoming message", - nodeDetails = "Delays messages for configurable period.", + nodeDetails = "Delays messages for configurable period. Please note, this node acknowledges the message from the current queue (message will be removed from queue)", icon = "pause", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeMsgDelayConfig" From e5fe102b3a46e2458ea1f1e04e36e78451c39bcb Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 29 Apr 2020 17:44:12 +0300 Subject: [PATCH 133/196] Fix limit to set coordinate --- .../resources/public/static/rulenode/rulenode-core-config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js index f848a91875..bed250d219 100644 --- a/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js +++ b/rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js @@ -1,6 +1,6 @@ !function(e){function t(a){if(n[a])return n[a].exports;var i=n[a]={exports:{},id:a,loaded:!1};return e[a].call(i.exports,i,i.exports,t),i.loaded=!0,i.exports}var n={};return t.m=e,t.c=n,t.p="/static/",t(0)}(function(e){for(var t in e)if(Object.prototype.hasOwnProperty.call(e,t))switch(typeof e[t]){case"function":break;case"object":e[t]=function(t){var n=t.slice(1),a=e[t[0]];return function(e,t,i){a.apply(this,[e,t,i].concat(n))}}(e[t]);break;default:e[t]=e[e[t]]}return e}([function(e,t,n){e.exports=n(105)},function(e,t){},1,1,1,1,function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
 {{ 'tb.rulenode.create-customer-if-not-exists' | translate }}
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports='
{{scope.name | translate}}
'},function(e,t){e.exports="
tb.rulenode.select-queue-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-details-function' | translate }}
{{ 'tb.rulenode.use-message-alarm-data' | translate }}
tb.rulenode.alarm-type-required
tb.rulenode.entity-type-pattern-hint
 {{ severity.name | translate}}
tb.rulenode.alarm-severity-required
{{ 'tb.rulenode.propagate' | translate }}
tb.rulenode.relation-types-list-hint
"},function(e,t){e.exports="
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.entity-type-pattern-required
tb.rulenode.entity-type-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
 {{ 'tb.rulenode.create-entity-if-not-exists' | translate }}
tb.rulenode.create-entity-if-not-exists-hint
{{ 'tb.rulenode.remove-current-relations' | translate }}
tb.rulenode.remove-current-relations-hint
{{ 'tb.rulenode.change-originator-to-related-entity' | translate }}
tb.rulenode.change-originator-to-related-entity-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'tb.rulenode.delete-relation-to-specific-entity' | translate }}
tb.rulenode.delete-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
tb.rulenode.entity-name-pattern-required
tb.rulenode.entity-name-pattern-hint
tb.rulenode.relation-type-pattern-required
tb.rulenode.relation-type-pattern-hint
tb.rulenode.entity-cache-expiration-required
tb.rulenode.entity-cache-expiration-range
tb.rulenode.entity-cache-expiration-hint
"},function(e,t){e.exports="
tb.rulenode.message-count-required
tb.rulenode.min-message-count-message
tb.rulenode.period-seconds-required
tb.rulenode.min-period-seconds-message
{{ 'tb.rulenode.test-generator-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
tb.rulenode.min-inside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.min-outside-duration-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
'},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.bootstrap-servers-required
tb.rulenode.min-retries-message
tb.rulenode.min-batch-size-bytes-message
tb.rulenode.min-linger-ms-message
tb.rulenode.min-buffer-memory-bytes-message
 {{ ackValue }}
tb.rulenode.key-serializer-required
tb.rulenode.value-serializer-required
 {{ \'tb.rulenode.add-metadata-key-values-as-kafka-headers\' | translate }}
tb.rulenode.add-metadata-key-values-as-kafka-headers-hint
{{charset.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-to-string-function' | translate }}
"},function(e,t){e.exports='
tb.rulenode.topic-pattern-required
tb.rulenode.mqtt-topic-pattern-hint
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
tb.rulenode.connect-timeout-required
tb.rulenode.connect-timeout-range
tb.rulenode.connect-timeout-range
{{ \'tb.rulenode.clean-session\' | translate }} {{ \'tb.rulenode.enable-ssl\' | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{ \'tb.rulenode.credentials\' | translate }}
{{ ruleNodeTypes.mqttCredentialTypes[configuration.credentials.type].name | translate }}
{{credentialsValue.name | translate}}
tb.rulenode.credentials-type-required
tb.rulenode.username-required
tb.rulenode.password-required
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.caCertFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.certFileName}}
{{ \'action.remove\' | translate }} close
tb.rulenode.no-file
{{configuration.credentials.privateKeyFileName}}
'; },function(e,t){e.exports="
tb.rulenode.interval-seconds-required
tb.rulenode.min-interval-seconds-message
tb.rulenode.output-timeseries-key-prefix-required
"},function(e,t){e.exports='
{{ \'tb.rulenode.use-metadata-period-in-seconds-patterns\' | translate }}
tb.rulenode.use-metadata-period-in-seconds-patterns-hint
tb.rulenode.period-seconds-required
tb.rulenode.min-period-0-seconds-message
tb.rulenode.period-in-seconds-pattern-required
tb.rulenode.period-in-seconds-pattern-hint
tb.rulenode.max-pending-messages-required
tb.rulenode.max-pending-messages-range
tb.rulenode.max-pending-messages-range
'},function(e,t){e.exports="
tb.rulenode.gcp-project-id-required
tb.rulenode.pubsub-topic-name-required
{{ 'action.remove' | translate }} close
tb.rulenode.no-file
{{configuration.serviceAccountKeyFileName}}
tb.rulenode.message-attributes-hint
"},function(e,t){e.exports='
{{ property }}
tb.rulenode.host-required
tb.rulenode.port-required
tb.rulenode.port-range
tb.rulenode.port-range
{{ \'tb.rulenode.automatic-recovery\' | translate }}
tb.rulenode.min-connection-timeout-ms-message
tb.rulenode.min-handshake-timeout-ms-message
'},function(e,t){e.exports='
tb.rulenode.endpoint-url-pattern-required
tb.rulenode.endpoint-url-pattern-hint
 {{ type }} {{ \'tb.rulenode.use-simple-client-http-factory\' | translate }}
tb.rulenode.read-timeout-hint
tb.rulenode.max-parallel-requests-count-hint
tb.rulenode.headers-hint
{{ \'tb.rulenode.use-redis-queue\' | translate }}
{{ \'tb.rulenode.trim-redis-queue\' | translate }}
'},function(e,t){e.exports="
"},function(e,t){e.exports="
tb.rulenode.timeout-required
tb.rulenode.min-timeout-message
"},function(e,t){e.exports='
tb.rulenode.custom-table-name-required
tb.rulenode.custom-table-hint
'},function(e,t){e.exports='
{{ \'tb.rulenode.use-system-smtp-settings\' | translate }}
{{smtpProtocol.toUpperCase()}}
tb.rulenode.smtp-host-required
tb.rulenode.smtp-port-required
tb.rulenode.smtp-port-range
tb.rulenode.smtp-port-range
tb.rulenode.timeout-required
tb.rulenode.min-timeout-msec-message
 {{ \'tb.rulenode.enable-tls\' | translate }} {{tlsVersion}}
'},function(e,t){e.exports="
tb.rulenode.topic-arn-pattern-required
tb.rulenode.topic-arn-pattern-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
"},function(e,t){e.exports='
{{ type.name | translate }}
tb.rulenode.queue-url-pattern-required
tb.rulenode.queue-url-pattern-hint
tb.rulenode.min-delay-seconds-message
tb.rulenode.max-delay-seconds-message
tb.rulenode.message-attributes-hint
tb.rulenode.aws-access-key-id-required
tb.rulenode.aws-secret-access-key-required
tb.rulenode.aws-region-required
'},function(e,t){e.exports="
tb.rulenode.default-ttl-required
tb.rulenode.min-default-ttl-message
"},function(e,t){e.exports="
tb.rulenode.customer-name-pattern-required
tb.rulenode.customer-name-pattern-hint
tb.rulenode.customer-cache-expiration-required
tb.rulenode.customer-cache-expiration-range
tb.rulenode.customer-cache-expiration-hint
"},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-type
device.device-types
"},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
{{\'tb.rulenode.entity-details-\'+item.toLowerCase() | translate}} tb.rulenode.no-entity-details-matching {{\'tb.rulenode.entity-details-\'+$chip.toLowerCase() | translate}} {{ \'tb.rulenode.add-to-metadata\' | translate }}
tb.rulenode.add-to-metadata-hint
'},function(e,t){e.exports='
{{ type }}
tb.rulenode.fetch-mode-hint
{{ type }}
tb.rulenode.order-by-hint
tb.rulenode.limit-hint
{{ \'tb.rulenode.use-metadata-interval-patterns\' | translate }}
tb.rulenode.use-metadata-interval-patterns-hint
tb.rulenode.start-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.end-interval-value-required
tb.rulenode.time-value-range
tb.rulenode.time-value-range
 {{timeUnit.name | translate}}
tb.rulenode.start-interval-pattern-required
tb.rulenode.start-interval-pattern-hint
tb.rulenode.end-interval-pattern-required
tb.rulenode.end-interval-pattern-hint
'; -},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},32,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.serviceType="TB_RULE_ENGINE",n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(77),r=a(i),o=n(54),l=a(o),s=n(60),d=a(s),u=n(57),c=a(u),m=n(56),g=a(m),p=n(64),f=a(p),b=n(71),v=a(b),y=n(72),h=a(y),q=n(70),k=a(q),x=n(63),$=a(x),T=n(75),C=a(T),w=n(76),M=a(w),N=n(69),S=a(N),_=n(65),F=a(_),E=n(74),P=a(E),A=n(67),V=a(A),I=n(66),j=a(I),O=n(53),D=a(O),L=n(78),R=a(L),K=n(59),U=a(K),z=n(58),H=a(z),B=n(73),G=a(B),Y=n(61),Q=a(Y),W=n(68),J=a(W),Z=n(55),X=a(Z);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",S.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",j.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).directive("tbActionNodeCheckPointConfig",X.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){ -var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader;t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(86),r=a(i),o=n(87),l=a(o),s=n(82),d=a(s),u=n(88),c=a(u),m=n(81),g=a(m),p=n(89),f=a(p),b=n(84),v=a(b),y=n(83),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(43),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(97),r=a(i),o=n(95),l=a(o),s=n(98),d=a(s),u=n(92),c=a(u),m=n(96),g=a(m),p=n(91),f=a(p),b=n(93),v=a(b),y=n(90),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(101),r=a(i),o=n(103),l=a(o),s=n(104),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(52),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(108),r=a(i),o=n(94),l=a(o),s=n(85),d=a(s),u=n(102),c=a(u),m=n(62),g=a(m),p=n(80),f=a(p),b=n(100),v=a(b),y=n(79),h=a(y),q=n(99),k=a(q),x=n(107),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required", +},function(e,t){e.exports='
{{ \'tb.rulenode.tell-failure-if-absent\' | translate }}
tb.rulenode.tell-failure-if-absent-hint
{{ \'tb.rulenode.get-latest-value-with-ts\' | translate }}
tb.rulenode.get-latest-value-with-ts-hint
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.latest-telemetry' | translate }}
"},32,function(e,t){e.exports="
{{'alarm.display-status.' + item | translate}} {{'alarm.display-status.' + $chip | translate}}
"},function(e,t){e.exports='
tb.rulenode.separator-hint
tb.rulenode.separator-hint
{{ \'tb.rulenode.check-all-keys\' | translate }}
tb.rulenode.check-all-keys-hint
'},function(e,t){e.exports="
{{ 'tb.rulenode.check-relation-to-specific-entity' | translate }}
tb.rulenode.check-relation-hint
{{ ('relation.search-direction.' + direction) | translate}}
"},function(e,t){e.exports='
tb.rulenode.latitude-key-name-required
tb.rulenode.longitude-key-name-required
 {{ \'tb.rulenode.fetch-perimeter-info-from-message-metadata\' | translate }}
{{ type.name | translate}}
tb.rulenode.circle-center-latitude-required
tb.rulenode.circle-center-longitude-required
tb.rulenode.range-required
{{ type.name | translate}}
tb.rulenode.polygon-definition-required
tb.rulenode.polygon-definition-hint
'},function(e,t){e.exports='
{{item}}
tb.rulenode.no-message-types-found
tb.rulenode.no-message-type-matching tb.rulenode.create-new-message-type
{{$chip.name}}
'},function(e,t){e.exports='
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-filter-function' | translate }}
"},function(e,t){e.exports="
{{ 'tb.rulenode.test-switch-function' | translate }}
"},function(e,t){e.exports='
{{ keyText }} {{ valText }}  
{{keyRequiredText}}
{{valRequiredText}}
 {{ \'tb.key-val.remove-entry\' | translate }} close
{{ \'tb.key-val.add-entry\' | translate }} add {{ \'action.add\' | translate }}
'},function(e,t){e.exports="
{{ 'alias.last-level-relation' | translate}}
{{ ('relation.search-direction.' + direction) | translate}}
relation.relation-filters
"},function(e,t){e.exports='
{{ source.name | translate}}
'},function(e,t){e.exports="
{{ 'tb.rulenode.test-transformer-function' | translate }}
"},function(e,t){e.exports="
tb.rulenode.from-template-required
tb.rulenode.from-template-hint
tb.rulenode.to-template-required
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.mail-address-list-template-hint
tb.rulenode.subject-template-required
tb.rulenode.subject-template-hint
tb.rulenode.body-template-required
tb.rulenode.body-template-hint
"},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(6),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(7),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.serviceType="TB_RULE_ENGINE",n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(8),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(9),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.hasOwnProperty("relationTypes")||(i.configuration.relationTypes=[])},i.testDetailsBuildJs=function(e){var n=angular.copy(i.configuration.alarmDetailsBuildJs);a.testNodeScript(e,n,"json",t.instant("tb.rulenode.details")+"","Details",["msg","metadata","msgType"],i.ruleNodeId).then(function(e){i.configuration.alarmDetailsBuildJs=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(10),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(11),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(12),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n,a){var i=function(i,r,l,s){var d=o.default;r.html(d),i.types=n,i.originator=null,i.$watch("configuration",function(e,t){angular.equals(e,t)||s.$setViewValue(i.configuration)}),s.$render=function(){i.configuration=s.$viewValue,i.configuration.originatorId&&i.configuration.originatorType?i.originator={id:i.configuration.originatorId,entityType:i.configuration.originatorType}:i.originator=null,i.$watch("originator",function(e,t){angular.equals(e,t)||(i.originator?(s.$viewValue.originatorId=i.originator.id,s.$viewValue.originatorType=i.originator.entityType):(s.$viewValue.originatorId=null,s.$viewValue.originatorType=null))},!0)},i.testScript=function(e){var n=angular.copy(i.configuration.jsScript);a.testNodeScript(e,n,"generate",t.instant("tb.rulenode.generator")+"","Generate",["prevMsg","prevMetadata","prevMsgType"],i.ruleNodeId).then(function(e){i.configuration.jsScript=e,s.$setDirty()})},e(r.contents())(i)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:i}}i.$inject=["$compile","$translate","types","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(1);var r=n(13),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(14),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(77),r=a(i),o=n(54),l=a(o),s=n(60),d=a(s),u=n(57),c=a(u),m=n(56),g=a(m),p=n(64),f=a(p),b=n(71),v=a(b),y=n(72),h=a(y),q=n(70),k=a(q),x=n(63),$=a(x),T=n(75),C=a(T),w=n(76),M=a(w),N=n(69),S=a(N),_=n(65),F=a(_),E=n(74),P=a(E),A=n(67),V=a(A),I=n(66),j=a(I),O=n(53),D=a(O),L=n(78),R=a(L),K=n(59),U=a(K),z=n(58),H=a(z),B=n(73),G=a(B),Y=n(61),Q=a(Y),W=n(68),J=a(W),Z=n(55),X=a(Z);t.default=angular.module("thingsboard.ruleChain.config.action",[]).directive("tbActionNodeTimeseriesConfig",r.default).directive("tbActionNodeAttributesConfig",l.default).directive("tbActionNodeGeneratorConfig",d.default).directive("tbActionNodeCreateAlarmConfig",c.default).directive("tbActionNodeClearAlarmConfig",g.default).directive("tbActionNodeLogConfig",f.default).directive("tbActionNodeRpcReplyConfig",v.default).directive("tbActionNodeRpcRequestConfig",h.default).directive("tbActionNodeRestApiCallConfig",k.default).directive("tbActionNodeKafkaConfig",$.default).directive("tbActionNodeSnsConfig",C.default).directive("tbActionNodeSqsConfig",M.default).directive("tbActionNodeRabbitMqConfig",S.default).directive("tbActionNodeMqttConfig",F.default).directive("tbActionNodeSendEmailConfig",P.default).directive("tbActionNodeMsgDelayConfig",V.default).directive("tbActionNodeMsgCountConfig",j.default).directive("tbActionNodeAssignToCustomerConfig",D.default).directive("tbActionNodeUnAssignToCustomerConfig",R.default).directive("tbActionNodeDeleteRelationConfig",U.default).directive("tbActionNodeCreateRelationConfig",H.default).directive("tbActionNodeCustomTableConfig",G.default).directive("tbActionNodeGpsGeofencingConfig",Q.default).directive("tbActionNodePubSubConfig",J.default).directive("tbActionNodeCheckPointConfig",X.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ackValues=["all","-1","0","1"],n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue,n.configuration.hasOwnProperty("kafkaHeadersCharset")||(n.configuration.kafkaHeadersCharset="UTF-8")},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(15),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"string",t.instant("tb.rulenode.to-string")+"","ToString",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(16),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$mdExpansionPanel=t,a.ruleNodeTypes=n,a.credentialsTypeChanged=function(){var e=a.configuration.credentials.type;a.configuration.credentials={},a.configuration.credentials.type=e,a.updateValidity()},a.certFileAdded=function(e,t){var n=new FileReader;n.onload=function(n){a.$apply(function(){if(n.target.result){l.$setDirty();var i=n.target.result;i&&i.length>0&&("caCert"==t&&(a.configuration.credentials.caCertFileName=e.name,a.configuration.credentials.caCert=i),"privateKey"==t&&(a.configuration.credentials.privateKeyFileName=e.name,a.configuration.credentials.privateKey=i),"Cert"==t&&(a.configuration.credentials.certFileName=e.name,a.configuration.credentials.cert=i)),a.updateValidity()}})},n.readAsText(e.file)},a.clearCertFile=function(e){l.$setDirty(),"caCert"==e&&(a.configuration.credentials.caCertFileName=null,a.configuration.credentials.caCert=null),"privateKey"==e&&(a.configuration.credentials.privateKeyFileName=null,a.configuration.credentials.privateKey=null),"Cert"==e&&(a.configuration.credentials.certFileName=null,a.configuration.credentials.cert=null),a.updateValidity()},a.updateValidity=function(){var e=!0,t=a.configuration.credentials;t.type==n.mqttCredentialTypes["cert.PEM"].value&&(t.caCert&&t.cert&&t.privateKey||(e=!1)),l.$setValidity("Certs",e)},a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:a}}i.$inject=["$compile","$mdExpansionPanel","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i,n(2);var r=n(17),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(18),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){ +var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(19),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.serviceAccountFileAdded=function(e){var t=new FileReader;t.onload=function(t){n.$apply(function(){if(t.target.result){r.$setDirty();var a=t.target.result;a&&a.length>0&&(n.configuration.serviceAccountKeyFileName=e.name,n.configuration.serviceAccountKey=a),n.updateValidity()}})},t.readAsText(e.file)},n.clearServiceAccountFile=function(){r.$setDirty(),n.configuration.serviceAccountKeyFileName=null,n.configuration.serviceAccountKey=null,n.updateValidity()},n.updateValidity=function(){var e=!0,t=n.configuration;t.serviceAccountKeyFileName&&t.serviceAccountKey||(e=!1),r.$setValidity("SAKey",e)},n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(20),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.messageProperties=[null,"BASIC","TEXT_PLAIN","MINIMAL_BASIC","MINIMAL_PERSISTENT_BASIC","PERSISTENT_BASIC","PERSISTENT_TEXT_PLAIN"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(21),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(22),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(23),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(24),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(25),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.smtpProtocols=["smtp","smtps"],t.tlsVersions=["TLSv1","TLSv1.1","TLSv1.2","TLSv1.3"],t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(26),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(27),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(28),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(29),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(30),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(31),o=a(r)},function(e,t){"use strict";function n(e){var t=function(t,n,a,i){n.html("
"),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}n.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(32),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(33),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.entityDetailsList=[];for(var s in t.entityDetails){var d=s;n.entityDetailsList.push(d)}r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(34),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s);var d=186;a.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,d],a.ruleNodeTypes=n,a.aggPeriodTimeUnits={},a.aggPeriodTimeUnits.MINUTES=n.timeUnit.MINUTES,a.aggPeriodTimeUnits.HOURS=n.timeUnit.HOURS,a.aggPeriodTimeUnits.DAYS=n.timeUnit.DAYS,a.aggPeriodTimeUnits.MILLISECONDS=n.timeUnit.MILLISECONDS,a.aggPeriodTimeUnits.SECONDS=n.timeUnit.SECONDS,a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{},link:a}}i.$inject=["$compile","$mdConstant","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(35),o=a(r);n(3)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(86),r=a(i),o=n(87),l=a(o),s=n(82),d=a(s),u=n(88),c=a(u),m=n(81),g=a(m),p=n(89),f=a(p),b=n(84),v=a(b),y=n(83),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.enrichment",[]).directive("tbEnrichmentNodeOriginatorAttributesConfig",r.default).directive("tbEnrichmentNodeOriginatorFieldsConfig",l.default).directive("tbEnrichmentNodeDeviceAttributesConfig",d.default).directive("tbEnrichmentNodeRelatedAttributesConfig",c.default).directive("tbEnrichmentNodeCustomerAttributesConfig",g.default).directive("tbEnrichmentNodeTenantAttributesConfig",f.default).directive("tbEnrichmentNodeGetTelemetryFromDatabase",v.default).directive("tbEnrichmentNodeEntityDetailsConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(36),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(37),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(38),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(39),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),n.alarmStatusList=[];for(var s in t.alarmStatus)n.alarmStatusList.push(t.alarmStatus[s]);r.$render=function(){n.configuration=r.$viewValue},n.getAlarmStatusList=function(){return n.alarmStatusList.filter(function(e){return n.configuration.alarmStatusList.indexOf(e)===-1})},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(40),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l);var s=186;n.separatorKeys=[t.KEY_CODE.ENTER,t.KEY_CODE.COMMA,s],n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","$mdConstant"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(41),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(42),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{readonly:"=ngReadonly"},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(43),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(97),r=a(i),o=n(95),l=a(o),s=n(98),d=a(s),u=n(92),c=a(u),m=n(96),g=a(m),p=n(91),f=a(p),b=n(93),v=a(b),y=n(90),h=a(y);t.default=angular.module("thingsboard.ruleChain.config.filter",[]).directive("tbFilterNodeScriptConfig",r.default).directive("tbFilterNodeMessageTypeConfig",l.default).directive("tbFilterNodeSwitchConfig",d.default).directive("tbFilterNodeCheckRelationConfig",c.default).directive("tbFilterNodeOriginatorTypeConfig",g.default).directive("tbFilterNodeCheckMessageConfig",f.default).directive("tbFilterNodeGpsGeofencingConfig",v.default).directive("tbFilterNodeCheckAlarmStatusConfig",h.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){function s(){if(l.$viewValue){for(var e=[],t=0;t-1&&t.kvList.splice(e,1)}function l(){t.kvList||(t.kvList=[]),t.kvList.push({key:"",value:""})}function s(){var e={};t.kvList.forEach(function(t){t.key&&(e[t.key]=t.value)}),i.$setViewValue(e),d()}function d(){var e=!0;t.required&&!t.kvList.length&&(e=!1),i.$setValidity("kvMap",e)}var u=o.default;n.html(u),t.ngModelCtrl=i,t.removeKeyVal=r,t.addKeyVal=l,t.kvList=[],t.$watch("query",function(e,n){angular.equals(e,n)||i.$setViewValue(t.query)}),i.$render=function(){if(i.$viewValue){var e=i.$viewValue;t.kvList.length=0;for(var n in e)t.kvList.push({key:n,value:e[n]})}t.$watch("kvList",function(e,t){angular.equals(e,t)||s()},!0),d()},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{required:"=ngRequired",disabled:"=ngDisabled",requiredText:"=",keyText:"=",keyRequiredText:"=",valText:"=",valRequiredText:"="},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(48),o=a(r);n(5)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.types=t,n.$watch("query",function(e,t){angular.equals(e,t)||r.$setViewValue(n.query)}),r.$render=function(){n.query=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","types"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(49),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t){var n=function(n,a,i,r){var l=o.default;a.html(l),n.ruleNodeTypes=t,n.$watch("configuration",function(e,t){angular.equals(e,t)||r.$setViewValue(n.configuration)}),r.$render=function(){n.configuration=r.$viewValue},e(a.contents())(n)};return{restrict:"E",require:"^ngModel",scope:{},link:n}}i.$inject=["$compile","ruleNodeTypes"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(50),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(101),r=a(i),o=n(103),l=a(o),s=n(104),d=a(s);t.default=angular.module("thingsboard.ruleChain.config.transform",[]).directive("tbTransformationNodeChangeOriginatorConfig",r.default).directive("tbTransformationNodeScriptConfig",l.default).directive("tbTransformationNodeToEmailConfig",d.default).name},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e,t,n){var a=function(a,i,r,l){var s=o.default;i.html(s),a.$watch("configuration",function(e,t){angular.equals(e,t)||l.$setViewValue(a.configuration)}),l.$render=function(){a.configuration=l.$viewValue},a.testScript=function(e){var i=angular.copy(a.configuration.jsScript);n.testNodeScript(e,i,"update",t.instant("tb.rulenode.transformer")+"","Transform",["msg","metadata","msgType"],a.ruleNodeId).then(function(e){a.configuration.jsScript=e,l.$setDirty()})},e(i.contents())(a)};return{restrict:"E",require:"^ngModel",scope:{ruleNodeId:"="},link:a}}i.$inject=["$compile","$translate","ruleNodeScriptTest"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(51),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){var t=function(t,n,a,i){var r=o.default;n.html(r),t.$watch("configuration",function(e,n){angular.equals(e,n)||i.$setViewValue(t.configuration)}),i.$render=function(){t.configuration=i.$viewValue},e(n.contents())(t)};return{restrict:"E",require:"^ngModel",scope:{},link:t}}i.$inject=["$compile"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(52),o=a(r)},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var i=n(108),r=a(i),o=n(94),l=a(o),s=n(85),d=a(s),u=n(102),c=a(u),m=n(62),g=a(m),p=n(80),f=a(p),b=n(100),v=a(b),y=n(79),h=a(y),q=n(99),k=a(q),x=n(107),$=a(x);t.default=angular.module("thingsboard.ruleChain.config",[r.default,l.default,d.default,c.default,g.default]).directive("tbNodeEmptyConfig",f.default).directive("tbRelationsQueryConfig",v.default).directive("tbDeviceRelationsQueryConfig",h.default).directive("tbKvMapConfig",k.default).config($.default).name},function(e,t){"use strict";function n(e){var t={tb:{rulenode:{"create-entity-if-not-exists":"Create new entity if not exists","create-entity-if-not-exists-hint":"Create a new entity set above if it does not exist.","entity-name-pattern":"Name pattern","entity-name-pattern-required":"Name pattern is required","entity-name-pattern-hint":"Name pattern, use ${metaKeyName} to substitute variables from metadata","entity-type-pattern":"Type pattern","entity-type-pattern-required":"Type pattern is required","entity-type-pattern-hint":"Type pattern, use ${metaKeyName} to substitute variables from metadata","entity-cache-expiration":"Entities cache expiration time (sec)","entity-cache-expiration-hint":"Specifies maximum time interval allowed to store found entity records. 0 value means that records will never expire.","entity-cache-expiration-required":"Entities cache expiration time is required.","entity-cache-expiration-range":"Entities cache expiration time should be greater than or equal to 0.","customer-name-pattern":"Customer name pattern","customer-name-pattern-required":"Customer name pattern is required","create-customer-if-not-exists":"Create new customer if not exists","customer-cache-expiration":"Customers cache expiration time (sec)","customer-name-pattern-hint":"Customer name pattern, use ${metaKeyName} to substitute variables from metadata","customer-cache-expiration-hint":"Specifies maximum time interval allowed to store found customer records. 0 value means that records will never expire.","customer-cache-expiration-required":"Customers cache expiration time is required.","customer-cache-expiration-range":"Customers cache expiration time should be greater than or equal to 0.","start-interval":"Start Interval","end-interval":"End Interval","start-interval-time-unit":"Start Interval Time Unit","end-interval-time-unit":"End Interval Time Unit","fetch-mode":"Fetch mode","fetch-mode-hint":"If selected fetch mode 'ALL' you able to choose telemetry sampling order.","order-by":"Order by","order-by-hint":"Select to choose telemetry sampling order.",limit:"Limit","limit-hint":"Min limit value is 2, max - 1000. In case you want to fetch a single entry, select fetch mode 'FIRST' or 'LAST'.","time-unit-milliseconds":"Milliseconds","time-unit-seconds":"Seconds","time-unit-minutes":"Minutes","time-unit-hours":"Hours","time-unit-days":"Days","time-value-range":"Time value should be in a range from 1 to 2147483647'.","start-interval-value-required":"Start interval value is required.","end-interval-value-required":"End interval value is required.",filter:"Filter",switch:"Switch","message-type":"Message type","message-type-required":"Message type is required.","message-types-filter":"Message types filter","no-message-types-found":"No message types found","no-message-type-matching":"'{{messageType}}' not found.","create-new-message-type":"Create a new one!","message-types-required":"Message types are required.","client-attributes":"Client attributes","shared-attributes":"Shared attributes","server-attributes":"Server attributes","latest-timeseries":"Latest timeseries","data-keys":"Message data","metadata-keys":"Message metadata","relations-query":"Relations query","device-relations-query":"Device relations query","max-relation-level":"Max relation level","relation-type-pattern":"Relation type pattern","relation-type-pattern-hint":"Relation type pattern, use ${metaKeyName} to substitute variables from metadata","relation-type-pattern-required":"Relation type pattern is required","relation-types-list":"Relation types to propagate","relation-types-list-hint":"If Propagate relation types are not selected, alarms will be propagated without filtering by relation type.","unlimited-level":"Unlimited level","latest-telemetry":"Latest telemetry","attr-mapping":"Attributes mapping","source-attribute":"Source attribute","source-attribute-required":"Source attribute is required.","source-telemetry":"Source telemetry","source-telemetry-required":"Source telemetry is required.","target-attribute":"Target attribute","target-attribute-required":"Target attribute is required.","attr-mapping-required":"At least one attribute mapping should be specified.","fields-mapping":"Fields mapping","fields-mapping-required":"At least one field mapping should be specified.","source-field":"Source field","source-field-required":"Source field is required.","originator-source":"Originator source","originator-customer":"Customer","originator-tenant":"Tenant","originator-related":"Related","originator-alarm-originator":"Alarm Originator","clone-message":"Clone message",transform:"Transform","default-ttl":"Default TTL in seconds","default-ttl-required":"Default TTL is required.","min-default-ttl-message":"Only 0 minimum TTL is allowed.","message-count":"Message count (0 - unlimited)","message-count-required":"Message count is required.","min-message-count-message":"Only 0 minimum message count is allowed.","period-seconds":"Period in seconds","period-seconds-required":"Period is required.","use-metadata-period-in-seconds-patterns":"Use metadata period in seconds pattern","use-metadata-period-in-seconds-patterns-hint":"If selected, rule node use period in seconds interval pattern from message metadata assuming that intervals are in the seconds.","period-in-seconds-pattern":"Period in seconds metadata pattern","period-in-seconds-pattern-required":"Period in seconds pattern is required","period-in-seconds-pattern-hint":"Period in seconds pattern, use ${metaKeyName} to substitute variables from metadata","min-period-seconds-message":"Only 1 second minimum period is allowed.",originator:"Originator","message-body":"Message body","message-metadata":"Message metadata",generate:"Generate","test-generator-function":"Test generator function",generator:"Generator","test-filter-function":"Test filter function","test-switch-function":"Test switch function","test-transformer-function":"Test transformer function",transformer:"Transformer","alarm-create-condition":"Alarm create condition","test-condition-function":"Test condition function","alarm-clear-condition":"Alarm clear condition","alarm-details-builder":"Alarm details builder","test-details-function":"Test details function","alarm-type":"Alarm type","alarm-type-required":"Alarm type is required.","alarm-severity":"Alarm severity","alarm-severity-required":"Alarm severity is required","alarm-statuses-filter":"Alarm statuses filter","alarm-statuses-required":"Alarm statuses is required",propagate:"Propagate",condition:"Condition",details:"Details","to-string":"To string","test-to-string-function":"Test to string function","from-template":"From Template","from-template-required":"From Template is required","from-template-hint":"From address template, use ${metaKeyName} to substitute variables from metadata","to-template":"To Template","to-template-required":"To Template is required","mail-address-list-template-hint":"Comma separated address list, use ${metaKeyName} to substitute variables from metadata","cc-template":"Cc Template","bcc-template":"Bcc Template","subject-template":"Subject Template","subject-template-required":"Subject Template is required","subject-template-hint":"Mail subject template, use ${metaKeyName} to substitute variables from metadata","body-template":"Body Template","body-template-required":"Body Template is required","body-template-hint":"Mail body template, use ${metaKeyName} to substitute variables from metadata","request-id-metadata-attribute":"Request Id Metadata attribute name","timeout-sec":"Timeout in seconds","timeout-required":"Timeout is required","min-timeout-message":"Only 0 minimum timeout value is allowed.","endpoint-url-pattern":"Endpoint URL pattern","endpoint-url-pattern-required":"Endpoint URL pattern is required", "endpoint-url-pattern-hint":"HTTP URL address pattern, use ${metaKeyName} to substitute variables from metadata","request-method":"Request method","use-simple-client-http-factory":"Use simple client HTTP factory","read-timeout":"Read timeout in millis","read-timeout-hint":"The value of 0 means an infinite timeout","max-parallel-requests-count":"Max number of parallel requests","max-parallel-requests-count-hint":"The value of 0 specifies no limit in parallel processing",headers:"Headers","headers-hint":"Use ${metaKeyName} in header/value fields to substitute variables from metadata",header:"Header","header-required":"Header is required",value:"Value","value-required":"Value is required","topic-pattern":"Topic pattern","topic-pattern-required":"Topic pattern is required","mqtt-topic-pattern-hint":"MQTT topic pattern, use ${metaKeyName} to substitute variables from metadata","bootstrap-servers":"Bootstrap servers","bootstrap-servers-required":"Bootstrap servers value is required","other-properties":"Other properties",key:"Key","key-required":"Key is required",retries:"Automatically retry times if fails","min-retries-message":"Only 0 minimum retries is allowed.","batch-size-bytes":"Produces batch size in bytes","min-batch-size-bytes-message":"Only 0 minimum batch size is allowed.","linger-ms":"Time to buffer locally (ms)","min-linger-ms-message":"Only 0 ms minimum value is allowed.","buffer-memory-bytes":"Client buffer max size in bytes","min-buffer-memory-message":"Only 0 minimum buffer size is allowed.",acks:"Number of acknowledgments","key-serializer":"Key serializer","key-serializer-required":"Key serializer is required","value-serializer":"Value serializer","value-serializer-required":"Value serializer is required","topic-arn-pattern":"Topic ARN pattern","topic-arn-pattern-required":"Topic ARN pattern is required","topic-arn-pattern-hint":"Topic ARN pattern, use ${metaKeyName} to substitute variables from metadata","aws-access-key-id":"AWS Access Key ID","aws-access-key-id-required":"AWS Access Key ID is required","aws-secret-access-key":"AWS Secret Access Key","aws-secret-access-key-required":"AWS Secret Access Key is required","aws-region":"AWS Region","aws-region-required":"AWS Region is required","exchange-name-pattern":"Exchange name pattern","routing-key-pattern":"Routing key pattern","message-properties":"Message properties",host:"Host","host-required":"Host is required",port:"Port","port-required":"Port is required","port-range":"Port should be in a range from 1 to 65535.","virtual-host":"Virtual host",username:"Username",password:"Password","automatic-recovery":"Automatic recovery","connection-timeout-ms":"Connection timeout (ms)","min-connection-timeout-ms-message":"Only 0 ms minimum value is allowed.","handshake-timeout-ms":"Handshake timeout (ms)","min-handshake-timeout-ms-message":"Only 0 ms minimum value is allowed.","client-properties":"Client properties","queue-url-pattern":"Queue URL pattern","queue-url-pattern-required":"Queue URL pattern is required","queue-url-pattern-hint":"Queue URL pattern, use ${metaKeyName} to substitute variables from metadata","delay-seconds":"Delay (seconds)","min-delay-seconds-message":"Only 0 seconds minimum value is allowed.","max-delay-seconds-message":"Only 900 seconds maximum value is allowed.",name:"Name","name-required":"Name is required","queue-type":"Queue type","sqs-queue-standard":"Standard","sqs-queue-fifo":"FIFO","gcp-project-id":"GCP project ID","gcp-project-id-required":"GCP project ID is required","gcp-service-account-key":"GCP service account key file","gcp-service-account-key-required":"GCP service account key file is required","pubsub-topic-name":"Topic name","pubsub-topic-name-required":"Topic name is required","message-attributes":"Message attributes","message-attributes-hint":"Use ${metaKeyName} in name/value fields to substitute variables from metadata","connect-timeout":"Connection timeout (sec)","connect-timeout-required":"Connection timeout is required.","connect-timeout-range":"Connection timeout should be in a range from 1 to 200.","client-id":"Client ID","clean-session":"Clean session","enable-ssl":"Enable SSL",credentials:"Credentials","credentials-type":"Credentials type","credentials-type-required":"Credentials type is required.","credentials-anonymous":"Anonymous","credentials-basic":"Basic","credentials-pem":"PEM","username-required":"Username is required.","password-required":"Password is required.","ca-cert":"CA certificate file *","private-key":"Private key file *",cert:"Certificate file *","no-file":"No file selected.","drop-file":"Drop a file or click to select a file to upload.","private-key-password":"Private key password","use-system-smtp-settings":"Use system SMTP settings","use-metadata-interval-patterns":"Use metadata interval patterns","use-metadata-interval-patterns-hint":"If selected, rule node use start and end interval patterns from message metadata assuming that intervals are in the milliseconds.","use-message-alarm-data":"Use message alarm data","check-all-keys":"Check that all selected keys are present","check-all-keys-hint":"If selected, checks that all specified keys are present in the message data and metadata.","check-relation-to-specific-entity":"Check relation to specific entity","check-relation-hint":"Checks existence of relation to specific entity or to any entity based on direction and relation type.","delete-relation-to-specific-entity":"Delete relation to specific entity","delete-relation-hint":"Deletes relation from the originator of the incoming message to the specified entity or list of entities based on direction and type.","remove-current-relations":"Remove current relations","remove-current-relations-hint":"Removes current relations from the originator of the incoming message based on direction and type.","change-originator-to-related-entity":"Change originator to related entity","change-originator-to-related-entity-hint":"Used to process submitted message as a message from another entity.","start-interval-pattern":"Start interval pattern","end-interval-pattern":"End interval pattern","start-interval-pattern-required":"Start interval pattern is required","end-interval-pattern-required":"End interval pattern is required","start-interval-pattern-hint":"Start interval pattern, use ${metaKeyName} to substitute variables from metadata","end-interval-pattern-hint":"End interval pattern, use ${metaKeyName} to substitute variables from metadata","smtp-protocol":"Protocol","smtp-host":"SMTP host","smtp-host-required":"SMTP host is required.","smtp-port":"SMTP port","smtp-port-required":"You must supply a smtp port.","smtp-port-range":"SMTP port should be in a range from 1 to 65535.","timeout-msec":"Timeout ms","min-timeout-msec-message":"Only 0 ms minimum value is allowed.","enter-username":"Enter username","enter-password":"Enter password","enable-tls":"Enable TLS","tls-version":"TLS version","min-period-0-seconds-message":"Only 0 second minimum period is allowed.","max-pending-messages":"Maximum pending messages","max-pending-messages-required":"Maximum pending messages is required.","max-pending-messages-range":"Maximum pending messages should be in a range from 1 to 100000.","originator-types-filter":"Originator types filter","interval-seconds":"Interval in seconds","interval-seconds-required":"Interval is required.","min-interval-seconds-message":"Only 1 second minimum interval is allowed.","output-timeseries-key-prefix":"Output timeseries key prefix","output-timeseries-key-prefix-required":"Output timeseries key prefix required.","separator-hint":'You should press "enter" to complete field input.',"entity-details":"Select entity details:","entity-details-title":"Title","entity-details-country":"Country","entity-details-state":"State","entity-details-zip":"Zip","entity-details-address":"Address","entity-details-address2":"Address2","entity-details-additional_info":"Additional Info","entity-details-phone":"Phone","entity-details-email":"Email","add-to-metadata":"Add selected details to message metadata","add-to-metadata-hint":"If selected, adds the selected details keys to the message metadata instead of message data.","entity-details-list-empty":"No entity details selected.","no-entity-details-matching":"No entity details matching were found.","custom-table-name":"Custom table name","custom-table-name-required":"Table Name is required","custom-table-hint":"You should enter the table name without prefix 'cs_tb_'.","message-field":"Message field","message-field-required":"Message field is required.","table-col":"Table column","table-col-required":"Table column is required.","latitude-key-name":"Latitude key name","longitude-key-name":"Longitude key name","latitude-key-name-required":"Latitude key name is required.","longitude-key-name-required":"Longitude key name is required.","fetch-perimeter-info-from-message-metadata":"Fetch perimeter information from message metadata","perimeter-circle":"Circle","perimeter-polygon":"Polygon","perimeter-type":"Perimeter type","circle-center-latitude":"Center latitude","circle-center-latitude-required":"Center latitude is required.","circle-center-longitude":"Center longitude","circle-center-longitude-required":"Center longitude is required.","range-unit-meter":"Meter","range-unit-kilometer":"Kilometer","range-unit-foot":"Foot","range-unit-mile":"Mile","range-unit-nautical-mile":"Nautical mile","range-units":"Range units",range:"Range","range-required":"Range is required.","polygon-definition":"Polygon definition","polygon-definition-required":"Polygon definition is required.","polygon-definition-hint":"Please, use the following format for manual definition of polygon: [[lat1,lon1],[lat2,lon2], ... ,[latN,lonN]].","min-inside-duration":"Minimal inside duration","min-inside-duration-value-required":"Minimal inside duration is required","min-inside-duration-time-unit":"Minimal inside duration time unit","min-outside-duration":"Minimal outside duration","min-outside-duration-value-required":"Minimal outside duration is required","min-outside-duration-time-unit":"Minimal outside duration time unit","tell-failure-if-absent":"Tell Failure","tell-failure-if-absent-hint":'If at least one selected key doesn\'t exist the outbound message will report "Failure".',"get-latest-value-with-ts":"Fetch Latest telemetry with Timestamp","get-latest-value-with-ts-hint":'If selected, latest telemetry values will be added to the outbound message metadata with timestamp, e.g: "temp": "{\\"ts\\":1574329385897,\\"value\\":42}"',"use-redis-queue":"Use redis queue for message persistence","trim-redis-queue":"Trim redis queue","redis-queue-max-size":"Redis queue max size","add-metadata-key-values-as-kafka-headers":"Add Message metadata key-value pairs to Kafka record headers","add-metadata-key-values-as-kafka-headers-hint":"If selected, key-value pairs from message metadata will be added to the outgoing records headers as byte arrays with predefined charset encoding.","charset-encoding":"Charset encoding","charset-encoding-required":"Charset encoding is required.","charset-us-ascii":"US-ASCII","charset-iso-8859-1":"ISO-8859-1","charset-utf-8":"UTF-8","charset-utf-16be":"UTF-16BE","charset-utf-16le":"UTF-16LE","charset-utf-16":"UTF-16","select-queue-hint":"The queue name can be selected from a drop-down list or add a custom name."},"key-val":{key:"Key",value:"Value","remove-entry":"Remove entry","add-entry":"Add entry"}}};e.translations("en_US",t)}Object.defineProperty(t,"__esModule",{value:!0}),t.default=n},function(e,t,n){"use strict";function a(e){return e&&e.__esModule?e:{default:e}}function i(e){(0,o.default)(e)}i.$inject=["$translateProvider"],Object.defineProperty(t,"__esModule",{value:!0}),t.default=i;var r=n(106),o=a(r)},function(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=angular.module("thingsboard.ruleChain.config.types",[]).constant("ruleNodeTypes",{originatorSource:{CUSTOMER:{name:"tb.rulenode.originator-customer",value:"CUSTOMER"},TENANT:{name:"tb.rulenode.originator-tenant",value:"TENANT"},RELATED:{name:"tb.rulenode.originator-related",value:"RELATED"},ALARM_ORIGINATOR:{name:"tb.rulenode.originator-alarm-originator",value:"ALARM_ORIGINATOR"}},fetchModeType:["FIRST","LAST","ALL"],samplingOrder:["ASC","DESC"],httpRequestType:["GET","POST","PUT","DELETE"],entityDetails:{TITLE:{name:"tb.rulenode.entity-details-title",value:"TITLE"},COUNTRY:{name:"tb.rulenode.entity-details-country",value:"COUNTRY"},STATE:{name:"tb.rulenode.entity-details-state",value:"STATE"},ZIP:{name:"tb.rulenode.entity-details-zip",value:"ZIP"},ADDRESS:{name:"tb.rulenode.entity-details-address",value:"ADDRESS"},ADDRESS2:{name:"tb.rulenode.entity-details-address2",value:"ADDRESS2"},PHONE:{name:"tb.rulenode.entity-details-phone",value:"PHONE"},EMAIL:{name:"tb.rulenode.entity-details-email",value:"EMAIL"},ADDITIONAL_INFO:{name:"tb.rulenode.entity-details-additional_info",value:"ADDITIONAL_INFO"}},sqsQueueType:{STANDARD:{name:"tb.rulenode.sqs-queue-standard",value:"STANDARD"},FIFO:{name:"tb.rulenode.sqs-queue-fifo",value:"FIFO"}},perimeterType:{CIRCLE:{name:"tb.rulenode.perimeter-circle",value:"CIRCLE"},POLYGON:{name:"tb.rulenode.perimeter-polygon",value:"POLYGON"}},timeUnit:{MILLISECONDS:{value:"MILLISECONDS",name:"tb.rulenode.time-unit-milliseconds"},SECONDS:{value:"SECONDS",name:"tb.rulenode.time-unit-seconds"},MINUTES:{value:"MINUTES",name:"tb.rulenode.time-unit-minutes"},HOURS:{value:"HOURS",name:"tb.rulenode.time-unit-hours"},DAYS:{value:"DAYS",name:"tb.rulenode.time-unit-days"}},rangeUnit:{METER:{value:"METER",name:"tb.rulenode.range-unit-meter"},KILOMETER:{value:"KILOMETER",name:"tb.rulenode.range-unit-kilometer"},FOOT:{value:"FOOT",name:"tb.rulenode.range-unit-foot"},MILE:{value:"MILE",name:"tb.rulenode.range-unit-mile"},NAUTICAL_MILE:{value:"NAUTICAL_MILE",name:"tb.rulenode.range-unit-nautical-mile"}},mqttCredentialTypes:{anonymous:{value:"anonymous",name:"tb.rulenode.credentials-anonymous"},basic:{value:"basic",name:"tb.rulenode.credentials-basic"},"cert.PEM":{value:"cert.PEM",name:"tb.rulenode.credentials-pem"}},toBytesStandartCharsetTypes:{"US-ASCII":{value:"US-ASCII",name:"tb.rulenode.charset-us-ascii"},"ISO-8859-1":{value:"ISO-8859-1",name:"tb.rulenode.charset-iso-8859-1"},"UTF-8":{value:"UTF-8",name:"tb.rulenode.charset-utf-8"},"UTF-16BE":{value:"UTF-16BE",name:"tb.rulenode.charset-utf-16be"},"UTF-16LE":{value:"UTF-16LE",name:"tb.rulenode.charset-utf-16le"},"UTF-16":{value:"UTF-16",name:"tb.rulenode.charset-utf-16"}}}).name}])); //# sourceMappingURL=rulenode-core-config.js.map \ No newline at end of file From 8e9fb463619d7d4f0933711d0cd05048af33102f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Apr 2020 19:52:18 +0300 Subject: [PATCH 134/196] Fixed microservices --- .../server/coap/ThingsboardCoapTransportApplication.java | 2 +- .../server/http/ThingsboardHttpTransportApplication.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java index ec596db10d..5d35802655 100644 --- a/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java +++ b/transport/coap/src/main/java/org/thingsboard/server/coap/ThingsboardCoapTransportApplication.java @@ -26,7 +26,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue.kafka"}) +@ComponentScan({"org.thingsboard.server.coap", "org.thingsboard.server.common", "org.thingsboard.server.transport.coap", "org.thingsboard.server.queue"}) public class ThingsboardCoapTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java index f44af2105e..3aa2d7c0b8 100644 --- a/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java +++ b/transport/http/src/main/java/org/thingsboard/server/http/ThingsboardHttpTransportApplication.java @@ -24,7 +24,7 @@ import java.util.Arrays; @SpringBootApplication @EnableAsync -@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue.kafka"}) +@ComponentScan({"org.thingsboard.server.http", "org.thingsboard.server.common", "org.thingsboard.server.transport.http", "org.thingsboard.server.queue"}) public class ThingsboardHttpTransportApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; From 1599b24c3a748599a84f877987f452579c997ccf Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Wed, 29 Apr 2020 21:02:47 +0300 Subject: [PATCH 135/196] Develop/2.5 js executor (#2685) * moved kafka from service.js to own module * created awsSqs, pubSub, rabbitmq js-executors * revert RemoteJsInvokeService * revert thingsboard.yml * added queue settings to js-executor * refactored queue factories * added queue params to pubsub js-executor * azure service bus js-executor --- .../azure/servicebus/TbServiceBusAdmin.java | 8 +- .../provider/AwsSqsMonolithQueueFactory.java | 35 ++- .../provider/AwsSqsTbCoreQueueFactory.java | 28 ++- .../AwsSqsTbRuleEngineQueueFactory.java | 30 ++- .../provider/PubSubMonolithQueueFactory.java | 35 ++- .../provider/PubSubTbCoreQueueFactory.java | 38 +++- .../PubSubTbRuleEngineQueueFactory.java | 28 ++- .../RabbitMqMonolithQueueFactory.java | 35 ++- .../provider/RabbitMqTbCoreQueueFactory.java | 28 ++- .../RabbitMqTbRuleEngineQueueFactory.java | 28 ++- .../ServiceBusMonolithQueueFactory.java | 30 ++- .../ServiceBusTbCoreQueueFactory.java | 28 ++- .../ServiceBusTbRuleEngineQueueFactory.java | 28 ++- .../api/jsInvokeMessageProcessor.js | 55 ++--- .../config/custom-environment-variables.yml | 36 +++- msa/js-executor/config/default.yml | 26 ++- msa/js-executor/package.json | 7 +- msa/js-executor/queue/awsSqsTemplate.js | 199 ++++++++++++++++++ msa/js-executor/queue/kafkaTemplate.js | 181 ++++++++++++++++ msa/js-executor/queue/pubSubTemplate.js | 154 ++++++++++++++ msa/js-executor/queue/rabbitmqTemplate.js | 177 ++++++++++++++++ msa/js-executor/queue/serviceBusTemplate.js | 194 +++++++++++++++++ msa/js-executor/server.js | 115 +++------- .../rule/engine/delay/TbMsgDelayNode.java | 2 +- 24 files changed, 1375 insertions(+), 150 deletions(-) create mode 100644 msa/js-executor/queue/awsSqsTemplate.js create mode 100644 msa/js-executor/queue/kafkaTemplate.js create mode 100644 msa/js-executor/queue/pubSubTemplate.js create mode 100644 msa/js-executor/queue/rabbitmqTemplate.js create mode 100644 msa/js-executor/queue/serviceBusTemplate.js diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index 229d6b4244..f108de4a43 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.azure.servicebus; import com.microsoft.azure.servicebus.management.ManagementClient; import com.microsoft.azure.servicebus.management.QueueDescription; import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; +import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsException; import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; @@ -71,7 +72,12 @@ public class TbServiceBusAdmin implements TbQueueAdmin { client.createQueue(queueDescription); queues.add(topic); } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to create queue: [{}]", topic, e); + if (e instanceof MessagingEntityAlreadyExistsException) { + queues.add(topic); + log.info("[{}] queue already exists.", topic); + } else { + log.error("Failed to create queue: [{}]", topic, e); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 76ff04c238..c1d724c207 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -15,20 +15,25 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -40,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") @@ -52,6 +58,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -65,7 +72,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { + TbAwsSqsQueueAttributes sqsQueueAttributes, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -73,6 +81,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -144,8 +153,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> { + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index a7349091bd..4889a7182d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,11 +32,13 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -44,6 +48,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") @@ -55,6 +60,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -68,6 +74,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbAwsSqsQueueAttributes sqsQueueAttributes) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; @@ -75,6 +82,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -133,8 +141,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 3056eced2c..cf43bc5fe4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -27,11 +29,13 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -41,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") @@ -51,6 +56,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -61,12 +67,14 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes) { + TbAwsSqsQueueAttributes sqsQueueAttributes, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.sqsSettings = sqsSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -113,8 +121,26 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index af0b276c1a..29a60af1e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -30,6 +33,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -40,12 +44,14 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") @@ -58,6 +64,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportNotificationSettings transportNotificationSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -72,7 +79,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportNotificationSettings transportNotificationSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + TbPubSubSubscriptionSettings pubSubSubscriptionSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -86,6 +94,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -152,8 +161,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 0a23d58e46..7d85b5c3ba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,21 +30,24 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") @@ -53,6 +58,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin jsExecutorAdmin; @@ -64,12 +70,14 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueTransportApiSettings transportApiSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); @@ -127,8 +135,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 79501130ed..a6105fea0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,10 +41,12 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") @@ -52,6 +57,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbQueueRuleEngineSettings ruleEngineSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -63,12 +69,14 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -116,8 +124,26 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 72f45e5276..1f812ea13c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -34,12 +38,14 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") @@ -58,6 +64,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -65,7 +72,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments) { + TbRabbitMqQueueArguments queueArguments, + TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -79,6 +87,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -144,8 +153,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + @Bean + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index d84bbedbd2..65d582c4bc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,6 +32,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -40,10 +43,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") @@ -55,6 +60,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -68,6 +74,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments) { this.rabbitMqSettings = rabbitMqSettings; this.coreSettings = coreSettings; @@ -75,6 +82,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); @@ -133,8 +141,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index a18f427fab..26abf78164 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,10 +41,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") @@ -52,6 +57,7 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -62,12 +68,14 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbRabbitMqSettings rabbitMqSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.rabbitMqSettings = rabbitMqSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); @@ -114,8 +122,26 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 3c82e18d91..3db6496b7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -35,19 +37,20 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.kafka.TbKafkaAdmin; -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") @@ -60,6 +63,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -73,6 +77,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; @@ -81,6 +86,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -152,8 +158,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index bcb48630f9..a3ba577f06 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -34,15 +36,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") @@ -54,6 +59,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -67,6 +73,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; @@ -74,6 +81,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -132,8 +140,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 1f492596a1..e4f6dbfbb4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.queue.provider; +import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -32,15 +34,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") @@ -51,6 +56,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbServiceBusSettings serviceBusSettings; + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -61,12 +67,14 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbServiceBusSettings serviceBusSettings, + TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.serviceBusSettings = serviceBusSettings; + this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -113,8 +121,26 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override + @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - return null; + TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); + TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), + msg -> { + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); + }); + + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder + , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); + builder.queueAdmin(jsExecutorAdmin); + builder.requestTemplate(producer); + builder.responseTemplate(consumer); + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); + builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); + return builder.build(); } @PreDestroy diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index f0facf8cc1..79ea505bcf 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,7 +19,6 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; -let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -31,6 +30,7 @@ const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); function JsInvokeMessageProcessor(producer) { + console.log("Producer:", producer); this.producer = producer; this.executor = new JsExecutor(useSandbox); this.scriptMap = {}; @@ -40,24 +40,24 @@ function JsInvokeMessageProcessor(producer) { JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { - var requestId; - var responseTopic; + let requestId; + let responseTopic; try { - var request = JSON.parse(message.value.toString('utf8')); - headers = message.headers; - var buf = message.headers['requestId']; + let request = JSON.parse(Buffer.from(message.data).toString('utf8')); + let headers = message.headers; + let buf = Buffer.from(headers.data['requestId']); requestId = Utils.UUIDFromBuffer(buf); - buf = message.headers['responseTopic']; + buf = Buffer.from(headers.data['responseTopic']); responseTopic = buf.toString('utf8'); logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic); if (request.compileRequest) { - this.processCompileRequest(requestId, responseTopic, request.compileRequest); + this.processCompileRequest(requestId, responseTopic, headers, request.compileRequest); } else if (request.invokeRequest) { - this.processInvokeRequest(requestId, responseTopic, request.invokeRequest); + this.processInvokeRequest(requestId, responseTopic, headers, request.invokeRequest); } else if (request.releaseRequest) { - this.processReleaseRequest(requestId, responseTopic, request.releaseRequest); + this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest); } else { logger.error('[%s] Unknown request recevied!', requestId); } @@ -68,7 +68,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { } } -JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, compileRequest) { +JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, headers, compileRequest) { var scriptId = getScriptId(compileRequest); logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId); @@ -77,17 +77,17 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r this.cacheScript(scriptId, script); var compileResponse = createCompileResponse(scriptId, true); logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, compileResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse); }, (err) => { var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, compileResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse); } ); } -JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, invokeRequest) { +JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, headers, invokeRequest) { var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; @@ -103,7 +103,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re (result) => { var invokeResponse = createInvokeResponse(result, true); logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); }, (err) => { var errorCode; @@ -114,19 +114,19 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re } var invokeResponse = createInvokeResponse("", false, errorCode, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode); - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); } ) }, (err) => { var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR); - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); } ); } -JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, releaseRequest) { +JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); if (this.scriptMap[scriptId]) { @@ -138,28 +138,17 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r } var releaseResponse = createReleaseResponse(scriptId, true); logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, scriptId, null, null, releaseResponse); + this.sendResponse(requestId, responseTopic, headers, scriptId, null, null, releaseResponse); } -JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) { +JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, headers, scriptId, compileResponse, invokeResponse, releaseResponse) { var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); - this.producer.send( - { - topic: responseTopic, - messages: [ - { - key: scriptId, - value: rawResponse, - headers: headers - } - ] - } - ).then( + this.producer.send(responseTopic, scriptId, rawResponse, headers).then( () => {}, (err) => { if (err) { - logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message); + logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); logger.error(err.stack); } } diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 585dfe8adb..b290719739 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,11 +14,45 @@ # limitations under the License. # +service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) +request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" + +js: + response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS" + kafka: - request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" bootstrap: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" + replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" + topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" + +pubsub: + project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" + service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" + queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" + +aws_sqs: + access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" + secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" + region: "TB_QUEUE_AWS_SQS_REGION" + queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" + +rabbitmq: + host: "TB_QUEUE_RABBIT_MQ_HOST" + port: "TB_QUEUE_RABBIT_MQ_PORT" + virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" + username: "TB_QUEUE_RABBIT_MQ_USERNAME" + password: "TB_QUEUE_RABBIT_MQ_PASSWORD" + queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" + +service_bus: + namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" + sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" + sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" + max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" + queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" + logger: level: "LOGGER_LEVEL" path: "LOG_FOLDER" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 1290a8a429..3155b051dc 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,11 +14,35 @@ # limitations under the License. # +service-type: "kafka" +request_topic: "js_eval.requests" + +js: + response_poll_interval: "25" + kafka: - request_topic: "js.eval.requests" bootstrap: # Kafka Bootstrap Servers servers: "localhost:9092" + replication_factor: "1" + topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + +pubsub: + queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" + +aws_sqs: + queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" + +rabbitmq: + host: "localhost" + port: "5672" + virtual_host: "/" + username: "admin" + password: "password" + queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" + +service_bus: + queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" logger: level: "info" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 0cdafffef1..3dadac4f84 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -14,7 +14,12 @@ "dependencies": { "config": "^3.2.2", "js-yaml": "^3.12.0", - "kafkajs": "^1.11.0", + "kafkajs": "^1.12.0", + "@google-cloud/pubsub": "^1.7.1", + "aws-sdk": "^2.663.0", + "amqplib": "^0.5.5", + "@azure/service-bus": "^1.1.6", + "azure-sb": "^0.11.1", "long": "^4.0.0", "uuid-parse": "^1.0.0", "winston": "^3.0.0", diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js new file mode 100644 index 0000000000..0396824af1 --- /dev/null +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -0,0 +1,199 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('awsSqsTemplate'); + +const requestTopic = config.get('request_topic'); + +const accessKeyId = config.get('aws_sqs.access_key_id'); +const secretAccessKey = config.get('aws_sqs.secret_access_key'); +const region = config.get('aws_sqs.region'); +const AWS = require('aws-sdk'); +const queueProperties = config.get('aws_sqs.queue-properties'); +const poolInterval = config.get('js.response_poll_interval'); + +let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; +let sqsClient; +let requestQueueURL; +const queueUrls = new Map(); +let stopped = false; + +function AwsSqsProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + let msgBody = JSON.stringify( + { + key: scriptId, + data: [...rawResponse], + headers: headers + }); + + let responseQueueUrl = queueUrls.get(topicToSqsQueueName(responseTopic)); + + if (!responseQueueUrl) { + responseQueueUrl = await createQueue(responseTopic); + queueUrls.set(responseTopic, responseQueueUrl); + } + + let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; + + return new Promise((resolve, reject) => { + sqsClient.sendMessage(params, function (err, data) { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + AWS.config.update({accessKeyId: accessKeyId, secretAccessKey: secretAccessKey, region: region}); + + sqsClient = new AWS.SQS({apiVersion: '2012-11-05'}); + + const queues = await getQueues(); + + queues.forEach(queueUrl => { + const delimiterPosition = queueUrl.lastIndexOf('/'); + const queueName = queueUrl.substring(delimiterPosition + 1); + queueUrls.set(queueName, queueUrl); + }) + + parseQueueProperties(); + + requestQueueURL = queueUrls.get(topicToSqsQueueName(requestTopic)); + if (!requestQueueURL) { + requestQueueURL = await createQueue(requestTopic); + } + + const messageProcessor = new JsInvokeMessageProcessor(new AwsSqsProducer()); + + const params = { + MaxNumberOfMessages: 10, + QueueUrl: requestQueueURL, + WaitTimeSeconds: poolInterval / 1000 + }; + while (!stopped) { + const messages = await new Promise((resolve, reject) => { + sqsClient.receiveMessage(params, function (err, data) { + if (err) { + reject(err); + } else { + resolve(data.Messages); + } + }); + }); + + if (messages && messages.length > 0) { + const entries = []; + + messages.forEach(message => { + entries.push({ + Id: message.MessageId, + ReceiptHandle: message.ReceiptHandle + }); + messageProcessor.onJsInvokeMessage(JSON.parse(message.Body)); + }); + + const deleteBatch = { + QueueUrl: requestQueueURL, + Entries: entries + }; + sqsClient.deleteMessageBatch(deleteBatch, function (err, data) { + if (err) { + logger.error("Failed to delete messages from queue.", err.message); + } else { + //do nothing + } + }); + } + } + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +function createQueue(topic) { + let queueName = topicToSqsQueueName(topic); + let queueParams = {QueueName: queueName, Attributes: queueAttributes}; + + return new Promise((resolve, reject) => { + sqsClient.createQueue(queueParams, function (err, data) { + if (err) { + reject(err); + } else { + resolve(data.QueueUrl); + } + }); + }); +} + +function getQueues() { + return new Promise((resolve, reject) => { + sqsClient.listQueues(function (err, data) { + if (err) { + reject(err); + } else { + resolve(data.QueueUrls); + } + }); + }); +} + +function topicToSqsQueueName(topic) { + return topic.replace(/\./g, '_') + '.fifo'; +} + +function parseQueueProperties() { + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); +} + +process.on('exit', () => { + stopped = true; + logger.info('Aws Sqs client stopped.'); + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + if (sqsClient) { + logger.info('Stopping Aws Sqs client.') + try { + await sqsClient.close(); + logger.info('Aws Sqs client stopped.') + process.exit(status); + } catch (e) { + logger.info('Aws Sqs client stop error.'); + process.exit(status); + } + } else { + process.exit(status); + } +} diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js new file mode 100644 index 0000000000..699ca7be00 --- /dev/null +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -0,0 +1,181 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const {logLevel, Kafka} = require('kafkajs'); + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('kafkaTemplate'), + KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; +const replicationFactor = config.get('kafka.replication_factor'); +const topicProperties = config.get('kafka.topic-properties'); + +let kafkaClient; +let kafkaAdmin; +let consumer; +let producer; + +const topics = []; +const configEntries = []; + +function KafkaProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + + if (!topics.includes(responseTopic)) { + let createResponseTopicResult = await createTopic(responseTopic); + topics.push(responseTopic); + if (createResponseTopicResult) { + logger.info('Created new topic: %s', requestTopic); + } + } + + let headersData = headers.data; + headersData = Object.fromEntries(Object.entries(headersData).map(([key, value]) => [key, Buffer.from(value)])); + return producer.send( + { + topic: responseTopic, + messages: [ + { + key: scriptId, + value: rawResponse, + headers: headersData + } + ] + }); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + + const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); + const requestTopic = config.get('request_topic'); + + logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); + logger.info('Kafka Requests Topic: %s', requestTopic); + + kafkaClient = new Kafka({ + brokers: kafkaBootstrapServers.split(','), + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator + }); + + parseTopicProperties(); + + kafkaAdmin = kafkaClient.admin(); + await kafkaAdmin.connect(); + + let createRequestTopicResult = await createTopic(requestTopic); + + if (createRequestTopicResult) { + logger.info('Created new topic: %s', requestTopic); + } + + consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); + producer = kafkaClient.producer(); + const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); + await consumer.connect(); + await producer.connect(); + await consumer.subscribe({topic: requestTopic}); + + logger.info('Started ThingsBoard JavaScript Executor Microservice.'); + await consumer.run({ + eachMessage: async ({topic, partition, message}) => { + let headers = message.headers; + let key = message.key; + let data = message.value; + let msg = {}; + + headers = Object.fromEntries( + Object.entries(headers).map(([key, value]) => [key, [...value]])); + + msg.key = key.toString('utf8'); + msg.data = [...data]; + msg.headers = {data: headers} + messageProcessor.onJsInvokeMessage(msg); + }, + }); + + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +function createTopic(topic) { + return kafkaAdmin.createTopics({ + topics: [{ + topic: topic, + replicationFactor: replicationFactor, + configEntries: configEntries + }] + }); +} + +function parseTopicProperties() { + const props = topicProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + configEntries.push({name: p.substring(0, delimiterPosition), value: p.substring(delimiterPosition + 1)}); + }); +} + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + + if (kafkaAdmin) { + logger.info('Stopping Kafka Admin...'); + await kafkaAdmin.disconnect(); + logger.info('Kafka Admin stopped.'); + } + + if (consumer) { + logger.info('Stopping Kafka Consumer...'); + let _consumer = consumer; + consumer = null; + try { + await _consumer.disconnect(); + logger.info('Kafka Consumer stopped.'); + await disconnectProducer(); + process.exit(status); + } catch (e) { + logger.info('Kafka Consumer stop error.'); + await disconnectProducer(); + process.exit(status); + } + } else { + process.exit(status); + } +} + +async function disconnectProducer() { + if (producer) { + logger.info('Stopping Kafka Producer...'); + var _producer = producer; + producer = null; + try { + await _producer.disconnect(); + logger.info('Kafka Producer stopped.'); + } catch (e) { + logger.info('Kafka Producer stop error.'); + } + } +} diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js new file mode 100644 index 0000000000..17e1b56e1d --- /dev/null +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -0,0 +1,154 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('pubSubTemplate'); +const {PubSub} = require('@google-cloud/pubsub'); + +const projectId = config.get('pubsub.project_id'); +const credentials = JSON.parse(config.get('pubsub.service_account')); +const requestTopic = config.get('request_topic'); +const queueProperties = config.get('pubsub.queue-properties'); + +let pubSubClient; + +const topics = []; +const subscriptions = []; +const queueProps = []; + +function PubSubProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + + if (!(subscriptions.includes(responseTopic) && topics.includes(requestTopic))) { + await createTopic(requestTopic); + } + + let data = JSON.stringify( + { + key: scriptId, + data: [...rawResponse], + headers: headers + }); + let dataBuffer = Buffer.from(data); + return pubSubClient.topic(responseTopic).publish(dataBuffer); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + pubSubClient = new PubSub({projectId: projectId, credentials: credentials}); + + parseQueueProperties(); + + const topicList = await pubSubClient.getTopics(); + + if (topicList) { + topicList[0].forEach(topic => { + topics.push(getName(topic.name)); + }); + } + + const subscriptionList = await pubSubClient.getSubscriptions(); + + if (subscriptionList) { + topicList[0].forEach(sub => { + subscriptions.push(getName(sub.name)); + }); + } + + if (!(subscriptions.includes(requestTopic) && topics.includes(requestTopic))) { + await createTopic(requestTopic); + } + + const subscription = pubSubClient.subscription(requestTopic); + + const messageProcessor = new JsInvokeMessageProcessor(new PubSubProducer()); + + const messageHandler = message => { + + messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8'))); + message.ack(); + }; + + subscription.on('message', messageHandler); + + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +async function createTopic(topic) { + if (!topics.includes(topic)) { + await pubSubClient.createTopic(topic); + topics.push(topic); + logger.info('Created new Pub/Sub topic: %s', topic); + } + await createSubscription(topic) +} + +async function createSubscription(topic) { + if (!subscriptions.includes(topic)) { + await pubSubClient.createSubscription(topic, topic, { + topic: topic, + subscription: topic, + ackDeadlineSeconds: queueProps['ackDeadlineInSec'], + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} + }); + subscriptions.push(topic); + logger.info('Created new Pub/Sub subscription: %s', topic); + } +} + +function parseQueueProperties() { + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); +} + +function getName(fullName) { + const delimiterPosition = fullName.lastIndexOf('/'); + return fullName.substring(delimiterPosition + 1); +} + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + if (pubSubClient) { + logger.info('Stopping Pub/Sub client.') + try { + await pubSubClient.close(); + logger.info('Pub/Sub client stopped.') + process.exit(status); + } catch (e) { + logger.info('Pub/Sub client stop error.'); + process.exit(status); + } + } else { + process.exit(status); + } +} + diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js new file mode 100644 index 0000000000..1a2905c3a0 --- /dev/null +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -0,0 +1,177 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('rabbitmqTemplate'); + +const requestTopic = config.get('request_topic'); +const host = config.get('rabbitmq.host'); +const port = config.get('rabbitmq.port'); +const vhost = config.get('rabbitmq.virtual_host'); +const username = config.get('rabbitmq.username'); +const password = config.get('rabbitmq.password'); +const queueProperties = config.get('rabbitmq.queue-properties'); +const poolInterval = config.get('js.response_poll_interval'); + +const amqp = require('amqplib/callback_api'); + +let queueParams = {durable: false, exclusive: false, autoDelete: false}; +let connection; +let channel; +let stopped = false; +const responseTopics = []; + +function RabbitMqProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + + if (!responseTopics.includes(responseTopic)) { + await createQueue(responseTopic); + responseTopics.push(responseTopic); + } + + let data = JSON.stringify( + { + key: scriptId, + data: [...rawResponse], + headers: headers + }); + let dataBuffer = Buffer.from(data); + channel.sendToQueue(responseTopic, dataBuffer); + return new Promise((resolve, reject) => { + channel.waitForConfirms((err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + const url = `amqp://${host}:${port}${vhost}`; + + amqp.credentials.amqplain(username, password); + connection = await new Promise((resolve, reject) => { + amqp.connect(url, function (err, connection) { + if (err) { + reject(err); + } else { + resolve(connection); + } + }); + }); + + channel = await new Promise((resolve, reject) => { + connection.createConfirmChannel(function (err, channel) { + if (err) { + reject(err); + } else { + resolve(channel); + } + }); + }); + + parseQueueProperties(); + + await createQueue(requestTopic); + + const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer()); + + while (!stopped) { + let message = await new Promise((resolve, reject) => { + channel.get(requestTopic, {}, function (err, msg) { + if (err) { + reject(err); + } else { + resolve(msg); + } + }); + }); + + if (message) { + messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); + channel.ack(message); + } else { + await sleep(poolInterval); + } + } + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +function parseQueueProperties() { + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + queueParams[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); +} + +function createQueue(topic) { + return new Promise((resolve, reject) => { + channel.assertQueue(topic, queueParams, function (err) { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +} + +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + + if (channel) { + logger.info('Stopping RabbitMq chanel.') + await channel.close(); + logger.info('RabbitMq chanel stopped'); + } + + if (connection) { + logger.info('Stopping RabbitMq connection.') + try { + await connection.close(); + logger.info('RabbitMq client connection.') + process.exit(status); + } catch (e) { + logger.info('RabbitMq connection stop error.'); + process.exit(status); + } + } else { + process.exit(status); + } +} diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js new file mode 100644 index 0000000000..034921afb7 --- /dev/null +++ b/msa/js-executor/queue/serviceBusTemplate.js @@ -0,0 +1,194 @@ +/* + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const config = require('config'), + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('serviceBusTemplate'); +const {ServiceBusClient, ReceiveMode} = require("@azure/service-bus"); +const azure = require('azure-sb'); + +const requestTopic = config.get('request_topic'); +const namespaceName = config.get('service_bus.namespace_name'); +const sasKeyName = config.get('service_bus.sas_key_name'); +const sasKey = config.get('service_bus.sas_key'); +const queueProperties = config.get('service_bus.queue-properties'); + +let sbClient; +let receiverClient; +let receiver; +let serviceBusService; + +let queueOptions = {}; +const queues = []; +const senderMap = new Map(); + +function ServiceBusProducer() { + this.send = async (responseTopic, scriptId, rawResponse, headers) => { + if (!queues.includes(requestTopic)) { + await createQueueIfNotExist(requestTopic); + queues.push(requestTopic); + } + + let customSender = senderMap.get(responseTopic); + + if (!customSender) { + customSender = new CustomSender(responseTopic); + senderMap.set(responseTopic, customSender); + } + + let data = { + key: scriptId, + data: [...rawResponse], + headers: headers + }; + + return customSender.send({body: data}); + } +} + +function CustomSender(topic) { + this.queueClient = sbClient.createQueueClient(topic); + this.sender = this.queueClient.createSender(); + + this.send = async (message) => { + return this.sender.send(message); + } +} + +(async () => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + + const connectionString = `Endpoint=sb://${namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${sasKeyName};SharedAccessKey=${sasKey}`; + sbClient = ServiceBusClient.createFromConnectionString(connectionString); + serviceBusService = azure.createServiceBusService(connectionString); + + parseQueueProperties(); + + await new Promise((resolve, reject) => { + serviceBusService.listQueues((err, data) => { + if (err) { + reject(err); + } else { + data.forEach(queue => { + queues.push(queue.QueueName); + }); + resolve(); + } + }); + }); + + if (!queues.includes(requestTopic)) { + await createQueueIfNotExist(requestTopic); + queues.push(requestTopic); + } + + receiverClient = sbClient.createQueueClient(requestTopic); + receiver = receiverClient.createReceiver(ReceiveMode.peekLock); + + const messageProcessor = new JsInvokeMessageProcessor(new ServiceBusProducer()); + + const messageHandler = async (message) => { + if (message) { + messageProcessor.onJsInvokeMessage(message.body); + await message.complete(); + } + }; + const errorHandler = (error) => { + logger.error('Failed to receive message from queue.', error); + }; + receiver.registerMessageHandler(messageHandler, errorHandler); + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +async function createQueueIfNotExist(topic) { + return new Promise((resolve, reject) => { + serviceBusService.createQueueIfNotExists(topic, queueOptions, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +} + +function parseQueueProperties() { + let properties = {}; + const props = queueProperties.split(';'); + props.forEach(p => { + const delimiterPosition = p.indexOf(':'); + properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); + }); + queueOptions = { + MaxSizeInMegabytes: properties['maxSizeInMb'], + DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, + LockDuration: `PT${properties['lockDurationInSec']}S` + }; +} + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + logger.info('Stopping Azure Service Bus resources...') + if (receiver) { + try { + await receiver.close(); + } catch (e) { + + } + } + + if (receiverClient) { + try { + await receiverClient.close(); + } catch (e) { + + } + } + + senderMap.forEach((k, v) => { + try { + v.sender.close(); + } catch (e) { + + } + try { + v.queueClient.close(); + } catch (e) { + + } + }); + + if (sbClient) { + try { + sbClient.close(); + } catch (e) { + + } + } + logger.info('Azure Service Bus resources stopped.') + process.exit(status); +} \ No newline at end of file diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index f56e5bb766..58361016c4 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -13,89 +13,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const { logLevel, Kafka } = require('kafkajs'); -const config = require('config'), - JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'), - logger = require('./config/logger')._logger('main'), - KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator; - -var kafkaClient; -var consumer; -var producer; - -(async() => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - - const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); - const kafkaRequestTopic = config.get('kafka.request_topic'); - - logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); - logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); - - kafkaClient = new Kafka({ - brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator - }); - - consumer = kafkaClient.consumer({ groupId: 'js-executor-group' }); - producer = kafkaClient.producer(); - const messageProcessor = new JsInvokeMessageProcessor(producer); - await consumer.connect(); - await producer.connect(); - await consumer.subscribe({ topic: kafkaRequestTopic}); - - logger.info('Started ThingsBoard JavaScript Executor Microservice.'); - await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - messageProcessor.onJsInvokeMessage(message); - }, - }); - - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -process.on('exit', () => { - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - if (consumer) { - logger.info('Stopping Kafka Consumer...'); - var _consumer = consumer; - consumer = null; - try { - await _consumer.disconnect(); - logger.info('Kafka Consumer stopped.'); - await disconnectProducer(); - process.exit(status); - } catch (e) { - logger.info('Kafka Consumer stop error.'); - await disconnectProducer(); - process.exit(status); - } - } else { - process.exit(status); - } +const config = require('config'), logger = require('./config/logger')._logger('main'); + +const serviceType = config.get('service-type'); +switch (serviceType) { + case 'kafka': + logger.info('Starting kafka template.'); + require('./queue/kafkaTemplate'); + logger.info('kafka template started.'); + break; + case 'pubsub': + logger.info('Starting Pub/Sub template.') + require('./queue/pubSubTemplate'); + logger.info('Pub/Sub template started.') + break; + case 'aws-sqs': + logger.info('Starting Aws Sqs template.') + require('./queue/awsSqsTemplate'); + logger.info('Aws Sqs template started.') + break; + case 'rabbitmq': + logger.info('Starting RabbitMq template.') + require('./queue/rabbitmqTemplate'); + logger.info('RabbitMq template started.') + break; + case 'service-bus': + logger.info('Starting Azure Service Bus template.') + require('./queue/serviceBusTemplate'); + logger.info('Azure Service Bus template started.') + break; + default: + logger.error('Unknown service type: ', serviceType); + process.exit(-1); } -async function disconnectProducer() { - if (producer) { - logger.info('Stopping Kafka Producer...'); - var _producer = producer; - producer = null; - try { - await _producer.disconnect(); - logger.info('Kafka Producer stopped.'); - } catch (e) { - logger.info('Kafka Producer stop error.'); - } - } -} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index 1807c5c4fe..e2bb07fb6f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -41,7 +41,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; name = "delay", configClazz = TbMsgDelayNodeConfiguration.class, nodeDescription = "Delays incoming message", - nodeDetails = "Delays messages for configurable period.", + nodeDetails = "Delays messages for configurable period. Please note, this node acknowledges the message from the current queue (message will be removed from queue)", icon = "pause", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeMsgDelayConfig" From 0cae071e5abb727175f72b394c2f6c1ae1e1a4e1 Mon Sep 17 00:00:00 2001 From: VoBa Date: Thu, 30 Apr 2020 09:37:08 +0300 Subject: [PATCH 136/196] =?UTF-8?q?Make=20executable=20files=20to=20be=20a?= =?UTF-8?q?ble=20to=20run=20by=20non=20root=20and=20non=20thingsboa?= =?UTF-8?q?=E2=80=A6=20(#2698)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make executable files to be able to run by non root and non thingsboard user (to be in sync with OpenShift policies) * Added default yes to agreement --- msa/js-executor/docker/Dockerfile | 4 +++- msa/tb-node/docker/Dockerfile | 4 +++- msa/transport/coap/docker/Dockerfile | 4 +++- msa/transport/http/docker/Dockerfile | 4 +++- msa/transport/mqtt/docker/Dockerfile | 4 +++- msa/web-ui/docker/Dockerfile | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/msa/js-executor/docker/Dockerfile b/msa/js-executor/docker/Dockerfile index 276fd03b13..d210a4dbb3 100644 --- a/msa/js-executor/docker/Dockerfile +++ b/msa/js-executor/docker/Dockerfile @@ -21,10 +21,12 @@ COPY start-js-executor.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-js-executor.sh /usr/bin -RUN dpkg -i /tmp/${pkg.name}.deb +RUN yes | dpkg -i /tmp/${pkg.name}.deb RUN update-rc.d ${pkg.name} disable +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name} + USER ${pkg.user} CMD ["start-js-executor.sh"] diff --git a/msa/tb-node/docker/Dockerfile b/msa/tb-node/docker/Dockerfile index eee8330f15..b7a2dbf346 100644 --- a/msa/tb-node/docker/Dockerfile +++ b/msa/tb-node/docker/Dockerfile @@ -21,12 +21,14 @@ COPY start-tb-node.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-node.sh /usr/bin -RUN dpkg -i /tmp/${pkg.name}.deb +RUN yes | dpkg -i /tmp/${pkg.name}.deb RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : RUN chown -R ${pkg.user}:${pkg.user} /tmp +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + USER ${pkg.user} CMD ["start-tb-node.sh"] diff --git a/msa/transport/coap/docker/Dockerfile b/msa/transport/coap/docker/Dockerfile index 07cb0101b9..5f297cdc4d 100644 --- a/msa/transport/coap/docker/Dockerfile +++ b/msa/transport/coap/docker/Dockerfile @@ -21,10 +21,12 @@ COPY start-tb-coap-transport.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-coap-transport.sh /usr/bin -RUN dpkg -i /tmp/${pkg.name}.deb +RUN yes | dpkg -i /tmp/${pkg.name}.deb RUN update-rc.d ${pkg.name} disable +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + USER ${pkg.user} CMD ["start-tb-coap-transport.sh"] diff --git a/msa/transport/http/docker/Dockerfile b/msa/transport/http/docker/Dockerfile index b49cf204f8..32d3bdaf98 100644 --- a/msa/transport/http/docker/Dockerfile +++ b/msa/transport/http/docker/Dockerfile @@ -21,10 +21,12 @@ COPY start-tb-http-transport.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-http-transport.sh /usr/bin -RUN dpkg -i /tmp/${pkg.name}.deb +RUN yes | dpkg -i /tmp/${pkg.name}.deb RUN update-rc.d ${pkg.name} disable +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + USER ${pkg.user} CMD ["start-tb-http-transport.sh"] diff --git a/msa/transport/mqtt/docker/Dockerfile b/msa/transport/mqtt/docker/Dockerfile index 149911f8b5..cc3b90e570 100644 --- a/msa/transport/mqtt/docker/Dockerfile +++ b/msa/transport/mqtt/docker/Dockerfile @@ -21,10 +21,12 @@ COPY start-tb-mqtt-transport.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-mqtt-transport.sh /usr/bin -RUN dpkg -i /tmp/${pkg.name}.deb +RUN yes | dpkg -i /tmp/${pkg.name}.deb RUN update-rc.d ${pkg.name} disable +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + USER ${pkg.user} CMD ["start-tb-mqtt-transport.sh"] diff --git a/msa/web-ui/docker/Dockerfile b/msa/web-ui/docker/Dockerfile index 3609c289e4..7f6178111b 100644 --- a/msa/web-ui/docker/Dockerfile +++ b/msa/web-ui/docker/Dockerfile @@ -21,10 +21,12 @@ COPY start-web-ui.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-web-ui.sh /usr/bin -RUN dpkg -i /tmp/${pkg.name}.deb +RUN yes | dpkg -i /tmp/${pkg.name}.deb RUN update-rc.d ${pkg.name} disable +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name} + USER ${pkg.user} CMD ["start-web-ui.sh"] From b8a305ec57a783a087d8b4ba221a9be534f67ec1 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Thu, 30 Apr 2020 09:50:50 +0300 Subject: [PATCH 137/196] Fix position autocomplete (#2699) --- ui/src/app/components/gateway/gateway-config-select.scss | 6 ------ ui/src/app/components/gateway/gateway-form.tpl.html | 5 ++--- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/ui/src/app/components/gateway/gateway-config-select.scss b/ui/src/app/components/gateway/gateway-config-select.scss index 1c189a7279..da2e7ba432 100644 --- a/ui/src/app/components/gateway/gateway-config-select.scss +++ b/ui/src/app/components/gateway/gateway-config-select.scss @@ -27,9 +27,3 @@ .tb-gateway-autocomplete-container.md-virtual-repeat-container.md-autocomplete-suggestions-container{ z-index: 70; } - -md-autocomplete{ - md-input-container{ - margin-bottom: 0; - } -} diff --git a/ui/src/app/components/gateway/gateway-form.tpl.html b/ui/src/app/components/gateway/gateway-form.tpl.html index c8d44d1fcb..13603b2cfc 100644 --- a/ui/src/app/components/gateway/gateway-form.tpl.html +++ b/ui/src/app/components/gateway/gateway-form.tpl.html @@ -39,12 +39,11 @@ get_access_token="vm.getAccessToken" create-device="vm.createDevice"> - + - + From 7a47cd503b6878bd01e084f6c66b1bdc049a0014 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Thu, 30 Apr 2020 11:19:17 +0300 Subject: [PATCH 138/196] Revert "Develop/2.5 js executor (#2685)" This reverts commit 1599b24c3a748599a84f877987f452579c997ccf. --- .../azure/servicebus/TbServiceBusAdmin.java | 8 +- .../provider/AwsSqsMonolithQueueFactory.java | 35 +-- .../provider/AwsSqsTbCoreQueueFactory.java | 28 +-- .../AwsSqsTbRuleEngineQueueFactory.java | 30 +-- .../provider/PubSubMonolithQueueFactory.java | 35 +-- .../provider/PubSubTbCoreQueueFactory.java | 38 +--- .../PubSubTbRuleEngineQueueFactory.java | 28 +-- .../RabbitMqMonolithQueueFactory.java | 35 +-- .../provider/RabbitMqTbCoreQueueFactory.java | 28 +-- .../RabbitMqTbRuleEngineQueueFactory.java | 28 +-- .../ServiceBusMonolithQueueFactory.java | 30 +-- .../ServiceBusTbCoreQueueFactory.java | 28 +-- .../ServiceBusTbRuleEngineQueueFactory.java | 28 +-- .../api/jsInvokeMessageProcessor.js | 55 +++-- .../config/custom-environment-variables.yml | 36 +--- msa/js-executor/config/default.yml | 26 +-- msa/js-executor/package.json | 7 +- msa/js-executor/queue/awsSqsTemplate.js | 199 ------------------ msa/js-executor/queue/kafkaTemplate.js | 181 ---------------- msa/js-executor/queue/pubSubTemplate.js | 154 -------------- msa/js-executor/queue/rabbitmqTemplate.js | 177 ---------------- msa/js-executor/queue/serviceBusTemplate.js | 194 ----------------- msa/js-executor/server.js | 115 +++++++--- .../rule/engine/delay/TbMsgDelayNode.java | 2 +- 24 files changed, 150 insertions(+), 1375 deletions(-) delete mode 100644 msa/js-executor/queue/awsSqsTemplate.js delete mode 100644 msa/js-executor/queue/kafkaTemplate.js delete mode 100644 msa/js-executor/queue/pubSubTemplate.js delete mode 100644 msa/js-executor/queue/rabbitmqTemplate.js delete mode 100644 msa/js-executor/queue/serviceBusTemplate.js diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index f108de4a43..229d6b4244 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -18,7 +18,6 @@ package org.thingsboard.server.queue.azure.servicebus; import com.microsoft.azure.servicebus.management.ManagementClient; import com.microsoft.azure.servicebus.management.QueueDescription; import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsException; import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; @@ -72,12 +71,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { client.createQueue(queueDescription); queues.add(topic); } catch (ServiceBusException | InterruptedException e) { - if (e instanceof MessagingEntityAlreadyExistsException) { - queues.add(topic); - log.info("[{}] queue already exists.", topic); - } else { - log.error("Failed to create queue: [{}]", topic, e); - } + log.error("Failed to create queue: [{}]", topic, e); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index c1d724c207..76ff04c238 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -15,25 +15,20 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; @@ -45,7 +40,6 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") @@ -58,7 +52,6 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -72,8 +65,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -81,7 +73,6 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -153,26 +144,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, - jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> { - RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 4889a7182d..a7349091bd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -32,13 +30,11 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -48,7 +44,6 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") @@ -60,7 +55,6 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -74,7 +68,6 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbAwsSqsQueueAttributes sqsQueueAttributes) { this.sqsSettings = sqsSettings; this.coreSettings = coreSettings; @@ -82,7 +75,6 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -141,26 +133,8 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, - jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index cf43bc5fe4..3056eced2c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -29,13 +27,11 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; @@ -45,7 +41,6 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") @@ -56,7 +51,6 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbAwsSqsSettings sqsSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -67,14 +61,12 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbAwsSqsQueueAttributes sqsQueueAttributes) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.sqsSettings = sqsSettings; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); @@ -121,26 +113,8 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, - jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 29a60af1e2..af0b276c1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -15,13 +15,10 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -33,7 +30,6 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -44,14 +40,12 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") @@ -64,7 +58,6 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportNotificationSettings transportNotificationSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -79,8 +72,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportNotificationSettings transportNotificationSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -94,7 +86,6 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -161,26 +152,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 7d85b5c3ba..0a23d58e46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,24 +28,21 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") @@ -58,7 +53,6 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin jsExecutorAdmin; @@ -70,14 +64,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueTransportApiSettings transportApiSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); @@ -135,26 +127,8 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index a6105fea0c..79501130ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,7 +28,6 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -41,12 +38,10 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") @@ -57,7 +52,6 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbQueueRuleEngineSettings ruleEngineSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -69,14 +63,12 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -124,26 +116,8 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1f812ea13c..72f45e5276 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -15,19 +15,15 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -38,14 +34,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") @@ -64,7 +58,6 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -72,8 +65,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -87,7 +79,6 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - this.jsInvokeSettings = jsInvokeSettings; } @Override @@ -153,26 +144,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index 65d582c4bc..d84bbedbd2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -32,7 +30,6 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -43,12 +40,10 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") @@ -60,7 +55,6 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -74,7 +68,6 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments) { this.rabbitMqSettings = rabbitMqSettings; this.coreSettings = coreSettings; @@ -82,7 +75,6 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); @@ -141,26 +133,8 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 26abf78164..a18f427fab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,7 +28,6 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -41,12 +38,10 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") @@ -57,7 +52,6 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -68,14 +62,12 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbRabbitMqSettings rabbitMqSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbRabbitMqQueueArguments queueArguments) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.rabbitMqSettings = rabbitMqSettings; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); @@ -122,26 +114,8 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 3db6496b7d..3c82e18d91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -37,20 +35,19 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") @@ -63,7 +60,6 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -77,7 +73,6 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; @@ -86,7 +81,6 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -158,26 +152,8 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index a3ba577f06..bcb48630f9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -36,18 +34,15 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") @@ -59,7 +54,6 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final PartitionService partitionService; private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -73,7 +67,6 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.serviceBusSettings = serviceBusSettings; this.coreSettings = coreSettings; @@ -81,7 +74,6 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.partitionService = partitionService; this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -140,26 +132,8 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index e4f6dbfbb4..1f492596a1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.provider; -import com.google.protobuf.util.JsonFormat; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -34,18 +32,15 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") @@ -56,7 +51,6 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -67,14 +61,12 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact TbQueueRuleEngineSettings ruleEngineSettings, TbServiceInfoProvider serviceInfoProvider, TbServiceBusSettings serviceBusSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.partitionService = partitionService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.serviceBusSettings = serviceBusSettings; - this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); @@ -121,26 +113,8 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact } @Override - @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); + return null; } @PreDestroy diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 79ea505bcf..f0facf8cc1 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -19,6 +19,7 @@ const COMPILATION_ERROR = 0; const RUNTIME_ERROR = 1; const TIMEOUT_ERROR = 2; const UNRECOGNIZED = -1; +let headers; const config = require('config'), logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), @@ -30,7 +31,6 @@ const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); function JsInvokeMessageProcessor(producer) { - console.log("Producer:", producer); this.producer = producer; this.executor = new JsExecutor(useSandbox); this.scriptMap = {}; @@ -40,24 +40,24 @@ function JsInvokeMessageProcessor(producer) { JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { - let requestId; - let responseTopic; + var requestId; + var responseTopic; try { - let request = JSON.parse(Buffer.from(message.data).toString('utf8')); - let headers = message.headers; - let buf = Buffer.from(headers.data['requestId']); + var request = JSON.parse(message.value.toString('utf8')); + headers = message.headers; + var buf = message.headers['requestId']; requestId = Utils.UUIDFromBuffer(buf); - buf = Buffer.from(headers.data['responseTopic']); + buf = message.headers['responseTopic']; responseTopic = buf.toString('utf8'); logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic); if (request.compileRequest) { - this.processCompileRequest(requestId, responseTopic, headers, request.compileRequest); + this.processCompileRequest(requestId, responseTopic, request.compileRequest); } else if (request.invokeRequest) { - this.processInvokeRequest(requestId, responseTopic, headers, request.invokeRequest); + this.processInvokeRequest(requestId, responseTopic, request.invokeRequest); } else if (request.releaseRequest) { - this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest); + this.processReleaseRequest(requestId, responseTopic, request.releaseRequest); } else { logger.error('[%s] Unknown request recevied!', requestId); } @@ -68,7 +68,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { } } -JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, headers, compileRequest) { +JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, compileRequest) { var scriptId = getScriptId(compileRequest); logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId); @@ -77,17 +77,17 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r this.cacheScript(scriptId, script); var compileResponse = createCompileResponse(scriptId, true); logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse); + this.sendResponse(requestId, responseTopic, scriptId, compileResponse); }, (err) => { var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse); + this.sendResponse(requestId, responseTopic, scriptId, compileResponse); } ); } -JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, headers, invokeRequest) { +JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, invokeRequest) { var scriptId = getScriptId(invokeRequest); logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); this.executedScriptsCounter++; @@ -103,7 +103,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re (result) => { var invokeResponse = createInvokeResponse(result, true); logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); }, (err) => { var errorCode; @@ -114,19 +114,19 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re } var invokeResponse = createInvokeResponse("", false, errorCode, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode); - this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); } ) }, (err) => { var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR); - this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse); + this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); } ); } -JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) { +JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, releaseRequest) { var scriptId = getScriptId(releaseRequest); logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); if (this.scriptMap[scriptId]) { @@ -138,17 +138,28 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r } var releaseResponse = createReleaseResponse(scriptId, true); logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId); - this.sendResponse(requestId, responseTopic, headers, scriptId, null, null, releaseResponse); + this.sendResponse(requestId, responseTopic, scriptId, null, null, releaseResponse); } -JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, headers, scriptId, compileResponse, invokeResponse, releaseResponse) { +JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) { var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); - this.producer.send(responseTopic, scriptId, rawResponse, headers).then( + this.producer.send( + { + topic: responseTopic, + messages: [ + { + key: scriptId, + value: rawResponse, + headers: headers + } + ] + } + ).then( () => {}, (err) => { if (err) { - logger.error('[%s] Failed to send response to queue: %s', requestId, err.message); + logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message); logger.error(err.stack); } } diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b290719739..585dfe8adb 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,45 +14,11 @@ # limitations under the License. # -service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) -request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" - -js: - response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS" - kafka: + request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" bootstrap: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" - replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" - topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" - -pubsub: - project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" - service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" - queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" - -aws_sqs: - access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" - secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" - region: "TB_QUEUE_AWS_SQS_REGION" - queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" - -rabbitmq: - host: "TB_QUEUE_RABBIT_MQ_HOST" - port: "TB_QUEUE_RABBIT_MQ_PORT" - virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" - username: "TB_QUEUE_RABBIT_MQ_USERNAME" - password: "TB_QUEUE_RABBIT_MQ_PASSWORD" - queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" - -service_bus: - namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" - sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" - sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" - max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" - queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" - logger: level: "LOGGER_LEVEL" path: "LOG_FOLDER" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 3155b051dc..1290a8a429 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,35 +14,11 @@ # limitations under the License. # -service-type: "kafka" -request_topic: "js_eval.requests" - -js: - response_poll_interval: "25" - kafka: + request_topic: "js.eval.requests" bootstrap: # Kafka Bootstrap Servers servers: "localhost:9092" - replication_factor: "1" - topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" - -pubsub: - queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" - -aws_sqs: - queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" - -rabbitmq: - host: "localhost" - port: "5672" - virtual_host: "/" - username: "admin" - password: "password" - queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" - -service_bus: - queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" logger: level: "info" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 3dadac4f84..0cdafffef1 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -14,12 +14,7 @@ "dependencies": { "config": "^3.2.2", "js-yaml": "^3.12.0", - "kafkajs": "^1.12.0", - "@google-cloud/pubsub": "^1.7.1", - "aws-sdk": "^2.663.0", - "amqplib": "^0.5.5", - "@azure/service-bus": "^1.1.6", - "azure-sb": "^0.11.1", + "kafkajs": "^1.11.0", "long": "^4.0.0", "uuid-parse": "^1.0.0", "winston": "^3.0.0", diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js deleted file mode 100644 index 0396824af1..0000000000 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -const config = require('config'), - JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), - logger = require('../config/logger')._logger('awsSqsTemplate'); - -const requestTopic = config.get('request_topic'); - -const accessKeyId = config.get('aws_sqs.access_key_id'); -const secretAccessKey = config.get('aws_sqs.secret_access_key'); -const region = config.get('aws_sqs.region'); -const AWS = require('aws-sdk'); -const queueProperties = config.get('aws_sqs.queue-properties'); -const poolInterval = config.get('js.response_poll_interval'); - -let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; -let sqsClient; -let requestQueueURL; -const queueUrls = new Map(); -let stopped = false; - -function AwsSqsProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - let msgBody = JSON.stringify( - { - key: scriptId, - data: [...rawResponse], - headers: headers - }); - - let responseQueueUrl = queueUrls.get(topicToSqsQueueName(responseTopic)); - - if (!responseQueueUrl) { - responseQueueUrl = await createQueue(responseTopic); - queueUrls.set(responseTopic, responseQueueUrl); - } - - let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; - - return new Promise((resolve, reject) => { - sqsClient.sendMessage(params, function (err, data) { - if (err) { - reject(err); - } else { - resolve(data); - } - }); - }); - } -} - -(async () => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - AWS.config.update({accessKeyId: accessKeyId, secretAccessKey: secretAccessKey, region: region}); - - sqsClient = new AWS.SQS({apiVersion: '2012-11-05'}); - - const queues = await getQueues(); - - queues.forEach(queueUrl => { - const delimiterPosition = queueUrl.lastIndexOf('/'); - const queueName = queueUrl.substring(delimiterPosition + 1); - queueUrls.set(queueName, queueUrl); - }) - - parseQueueProperties(); - - requestQueueURL = queueUrls.get(topicToSqsQueueName(requestTopic)); - if (!requestQueueURL) { - requestQueueURL = await createQueue(requestTopic); - } - - const messageProcessor = new JsInvokeMessageProcessor(new AwsSqsProducer()); - - const params = { - MaxNumberOfMessages: 10, - QueueUrl: requestQueueURL, - WaitTimeSeconds: poolInterval / 1000 - }; - while (!stopped) { - const messages = await new Promise((resolve, reject) => { - sqsClient.receiveMessage(params, function (err, data) { - if (err) { - reject(err); - } else { - resolve(data.Messages); - } - }); - }); - - if (messages && messages.length > 0) { - const entries = []; - - messages.forEach(message => { - entries.push({ - Id: message.MessageId, - ReceiptHandle: message.ReceiptHandle - }); - messageProcessor.onJsInvokeMessage(JSON.parse(message.Body)); - }); - - const deleteBatch = { - QueueUrl: requestQueueURL, - Entries: entries - }; - sqsClient.deleteMessageBatch(deleteBatch, function (err, data) { - if (err) { - logger.error("Failed to delete messages from queue.", err.message); - } else { - //do nothing - } - }); - } - } - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -function createQueue(topic) { - let queueName = topicToSqsQueueName(topic); - let queueParams = {QueueName: queueName, Attributes: queueAttributes}; - - return new Promise((resolve, reject) => { - sqsClient.createQueue(queueParams, function (err, data) { - if (err) { - reject(err); - } else { - resolve(data.QueueUrl); - } - }); - }); -} - -function getQueues() { - return new Promise((resolve, reject) => { - sqsClient.listQueues(function (err, data) { - if (err) { - reject(err); - } else { - resolve(data.QueueUrls); - } - }); - }); -} - -function topicToSqsQueueName(topic) { - return topic.replace(/\./g, '_') + '.fifo'; -} - -function parseQueueProperties() { - const props = queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); -} - -process.on('exit', () => { - stopped = true; - logger.info('Aws Sqs client stopped.'); - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - if (sqsClient) { - logger.info('Stopping Aws Sqs client.') - try { - await sqsClient.close(); - logger.info('Aws Sqs client stopped.') - process.exit(status); - } catch (e) { - logger.info('Aws Sqs client stop error.'); - process.exit(status); - } - } else { - process.exit(status); - } -} diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js deleted file mode 100644 index 699ca7be00..0000000000 --- a/msa/js-executor/queue/kafkaTemplate.js +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -const {logLevel, Kafka} = require('kafkajs'); - -const config = require('config'), - JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), - logger = require('../config/logger')._logger('kafkaTemplate'), - KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; -const replicationFactor = config.get('kafka.replication_factor'); -const topicProperties = config.get('kafka.topic-properties'); - -let kafkaClient; -let kafkaAdmin; -let consumer; -let producer; - -const topics = []; -const configEntries = []; - -function KafkaProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - - if (!topics.includes(responseTopic)) { - let createResponseTopicResult = await createTopic(responseTopic); - topics.push(responseTopic); - if (createResponseTopicResult) { - logger.info('Created new topic: %s', requestTopic); - } - } - - let headersData = headers.data; - headersData = Object.fromEntries(Object.entries(headersData).map(([key, value]) => [key, Buffer.from(value)])); - return producer.send( - { - topic: responseTopic, - messages: [ - { - key: scriptId, - value: rawResponse, - headers: headersData - } - ] - }); - } -} - -(async () => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - - const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); - const requestTopic = config.get('request_topic'); - - logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); - logger.info('Kafka Requests Topic: %s', requestTopic); - - kafkaClient = new Kafka({ - brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator - }); - - parseTopicProperties(); - - kafkaAdmin = kafkaClient.admin(); - await kafkaAdmin.connect(); - - let createRequestTopicResult = await createTopic(requestTopic); - - if (createRequestTopicResult) { - logger.info('Created new topic: %s', requestTopic); - } - - consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); - producer = kafkaClient.producer(); - const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer()); - await consumer.connect(); - await producer.connect(); - await consumer.subscribe({topic: requestTopic}); - - logger.info('Started ThingsBoard JavaScript Executor Microservice.'); - await consumer.run({ - eachMessage: async ({topic, partition, message}) => { - let headers = message.headers; - let key = message.key; - let data = message.value; - let msg = {}; - - headers = Object.fromEntries( - Object.entries(headers).map(([key, value]) => [key, [...value]])); - - msg.key = key.toString('utf8'); - msg.data = [...data]; - msg.headers = {data: headers} - messageProcessor.onJsInvokeMessage(msg); - }, - }); - - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -function createTopic(topic) { - return kafkaAdmin.createTopics({ - topics: [{ - topic: topic, - replicationFactor: replicationFactor, - configEntries: configEntries - }] - }); -} - -function parseTopicProperties() { - const props = topicProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - configEntries.push({name: p.substring(0, delimiterPosition), value: p.substring(delimiterPosition + 1)}); - }); -} - -process.on('exit', () => { - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - - if (kafkaAdmin) { - logger.info('Stopping Kafka Admin...'); - await kafkaAdmin.disconnect(); - logger.info('Kafka Admin stopped.'); - } - - if (consumer) { - logger.info('Stopping Kafka Consumer...'); - let _consumer = consumer; - consumer = null; - try { - await _consumer.disconnect(); - logger.info('Kafka Consumer stopped.'); - await disconnectProducer(); - process.exit(status); - } catch (e) { - logger.info('Kafka Consumer stop error.'); - await disconnectProducer(); - process.exit(status); - } - } else { - process.exit(status); - } -} - -async function disconnectProducer() { - if (producer) { - logger.info('Stopping Kafka Producer...'); - var _producer = producer; - producer = null; - try { - await _producer.disconnect(); - logger.info('Kafka Producer stopped.'); - } catch (e) { - logger.info('Kafka Producer stop error.'); - } - } -} diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js deleted file mode 100644 index 17e1b56e1d..0000000000 --- a/msa/js-executor/queue/pubSubTemplate.js +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -const config = require('config'), - JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), - logger = require('../config/logger')._logger('pubSubTemplate'); -const {PubSub} = require('@google-cloud/pubsub'); - -const projectId = config.get('pubsub.project_id'); -const credentials = JSON.parse(config.get('pubsub.service_account')); -const requestTopic = config.get('request_topic'); -const queueProperties = config.get('pubsub.queue-properties'); - -let pubSubClient; - -const topics = []; -const subscriptions = []; -const queueProps = []; - -function PubSubProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - - if (!(subscriptions.includes(responseTopic) && topics.includes(requestTopic))) { - await createTopic(requestTopic); - } - - let data = JSON.stringify( - { - key: scriptId, - data: [...rawResponse], - headers: headers - }); - let dataBuffer = Buffer.from(data); - return pubSubClient.topic(responseTopic).publish(dataBuffer); - } -} - -(async () => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - pubSubClient = new PubSub({projectId: projectId, credentials: credentials}); - - parseQueueProperties(); - - const topicList = await pubSubClient.getTopics(); - - if (topicList) { - topicList[0].forEach(topic => { - topics.push(getName(topic.name)); - }); - } - - const subscriptionList = await pubSubClient.getSubscriptions(); - - if (subscriptionList) { - topicList[0].forEach(sub => { - subscriptions.push(getName(sub.name)); - }); - } - - if (!(subscriptions.includes(requestTopic) && topics.includes(requestTopic))) { - await createTopic(requestTopic); - } - - const subscription = pubSubClient.subscription(requestTopic); - - const messageProcessor = new JsInvokeMessageProcessor(new PubSubProducer()); - - const messageHandler = message => { - - messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8'))); - message.ack(); - }; - - subscription.on('message', messageHandler); - - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -async function createTopic(topic) { - if (!topics.includes(topic)) { - await pubSubClient.createTopic(topic); - topics.push(topic); - logger.info('Created new Pub/Sub topic: %s', topic); - } - await createSubscription(topic) -} - -async function createSubscription(topic) { - if (!subscriptions.includes(topic)) { - await pubSubClient.createSubscription(topic, topic, { - topic: topic, - subscription: topic, - ackDeadlineSeconds: queueProps['ackDeadlineInSec'], - messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} - }); - subscriptions.push(topic); - logger.info('Created new Pub/Sub subscription: %s', topic); - } -} - -function parseQueueProperties() { - const props = queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); -} - -function getName(fullName) { - const delimiterPosition = fullName.lastIndexOf('/'); - return fullName.substring(delimiterPosition + 1); -} - -process.on('exit', () => { - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - if (pubSubClient) { - logger.info('Stopping Pub/Sub client.') - try { - await pubSubClient.close(); - logger.info('Pub/Sub client stopped.') - process.exit(status); - } catch (e) { - logger.info('Pub/Sub client stop error.'); - process.exit(status); - } - } else { - process.exit(status); - } -} - diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js deleted file mode 100644 index 1a2905c3a0..0000000000 --- a/msa/js-executor/queue/rabbitmqTemplate.js +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -const config = require('config'), - JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), - logger = require('../config/logger')._logger('rabbitmqTemplate'); - -const requestTopic = config.get('request_topic'); -const host = config.get('rabbitmq.host'); -const port = config.get('rabbitmq.port'); -const vhost = config.get('rabbitmq.virtual_host'); -const username = config.get('rabbitmq.username'); -const password = config.get('rabbitmq.password'); -const queueProperties = config.get('rabbitmq.queue-properties'); -const poolInterval = config.get('js.response_poll_interval'); - -const amqp = require('amqplib/callback_api'); - -let queueParams = {durable: false, exclusive: false, autoDelete: false}; -let connection; -let channel; -let stopped = false; -const responseTopics = []; - -function RabbitMqProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - - if (!responseTopics.includes(responseTopic)) { - await createQueue(responseTopic); - responseTopics.push(responseTopic); - } - - let data = JSON.stringify( - { - key: scriptId, - data: [...rawResponse], - headers: headers - }); - let dataBuffer = Buffer.from(data); - channel.sendToQueue(responseTopic, dataBuffer); - return new Promise((resolve, reject) => { - channel.waitForConfirms((err) => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }); - } -} - -(async () => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - const url = `amqp://${host}:${port}${vhost}`; - - amqp.credentials.amqplain(username, password); - connection = await new Promise((resolve, reject) => { - amqp.connect(url, function (err, connection) { - if (err) { - reject(err); - } else { - resolve(connection); - } - }); - }); - - channel = await new Promise((resolve, reject) => { - connection.createConfirmChannel(function (err, channel) { - if (err) { - reject(err); - } else { - resolve(channel); - } - }); - }); - - parseQueueProperties(); - - await createQueue(requestTopic); - - const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer()); - - while (!stopped) { - let message = await new Promise((resolve, reject) => { - channel.get(requestTopic, {}, function (err, msg) { - if (err) { - reject(err); - } else { - resolve(msg); - } - }); - }); - - if (message) { - messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); - channel.ack(message); - } else { - await sleep(poolInterval); - } - } - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -function parseQueueProperties() { - const props = queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - queueParams[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); -} - -function createQueue(topic) { - return new Promise((resolve, reject) => { - channel.assertQueue(topic, queueParams, function (err) { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }); -} - -function sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -process.on('exit', () => { - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - - if (channel) { - logger.info('Stopping RabbitMq chanel.') - await channel.close(); - logger.info('RabbitMq chanel stopped'); - } - - if (connection) { - logger.info('Stopping RabbitMq connection.') - try { - await connection.close(); - logger.info('RabbitMq client connection.') - process.exit(status); - } catch (e) { - logger.info('RabbitMq connection stop error.'); - process.exit(status); - } - } else { - process.exit(status); - } -} diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js deleted file mode 100644 index 034921afb7..0000000000 --- a/msa/js-executor/queue/serviceBusTemplate.js +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -const config = require('config'), - JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), - logger = require('../config/logger')._logger('serviceBusTemplate'); -const {ServiceBusClient, ReceiveMode} = require("@azure/service-bus"); -const azure = require('azure-sb'); - -const requestTopic = config.get('request_topic'); -const namespaceName = config.get('service_bus.namespace_name'); -const sasKeyName = config.get('service_bus.sas_key_name'); -const sasKey = config.get('service_bus.sas_key'); -const queueProperties = config.get('service_bus.queue-properties'); - -let sbClient; -let receiverClient; -let receiver; -let serviceBusService; - -let queueOptions = {}; -const queues = []; -const senderMap = new Map(); - -function ServiceBusProducer() { - this.send = async (responseTopic, scriptId, rawResponse, headers) => { - if (!queues.includes(requestTopic)) { - await createQueueIfNotExist(requestTopic); - queues.push(requestTopic); - } - - let customSender = senderMap.get(responseTopic); - - if (!customSender) { - customSender = new CustomSender(responseTopic); - senderMap.set(responseTopic, customSender); - } - - let data = { - key: scriptId, - data: [...rawResponse], - headers: headers - }; - - return customSender.send({body: data}); - } -} - -function CustomSender(topic) { - this.queueClient = sbClient.createQueueClient(topic); - this.sender = this.queueClient.createSender(); - - this.send = async (message) => { - return this.sender.send(message); - } -} - -(async () => { - try { - logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - - const connectionString = `Endpoint=sb://${namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${sasKeyName};SharedAccessKey=${sasKey}`; - sbClient = ServiceBusClient.createFromConnectionString(connectionString); - serviceBusService = azure.createServiceBusService(connectionString); - - parseQueueProperties(); - - await new Promise((resolve, reject) => { - serviceBusService.listQueues((err, data) => { - if (err) { - reject(err); - } else { - data.forEach(queue => { - queues.push(queue.QueueName); - }); - resolve(); - } - }); - }); - - if (!queues.includes(requestTopic)) { - await createQueueIfNotExist(requestTopic); - queues.push(requestTopic); - } - - receiverClient = sbClient.createQueueClient(requestTopic); - receiver = receiverClient.createReceiver(ReceiveMode.peekLock); - - const messageProcessor = new JsInvokeMessageProcessor(new ServiceBusProducer()); - - const messageHandler = async (message) => { - if (message) { - messageProcessor.onJsInvokeMessage(message.body); - await message.complete(); - } - }; - const errorHandler = (error) => { - logger.error('Failed to receive message from queue.', error); - }; - receiver.registerMessageHandler(messageHandler, errorHandler); - } catch (e) { - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - logger.error(e.stack); - exit(-1); - } -})(); - -async function createQueueIfNotExist(topic) { - return new Promise((resolve, reject) => { - serviceBusService.createQueueIfNotExists(topic, queueOptions, (err) => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }); -} - -function parseQueueProperties() { - let properties = {}; - const props = queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); - queueOptions = { - MaxSizeInMegabytes: properties['maxSizeInMb'], - DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, - LockDuration: `PT${properties['lockDurationInSec']}S` - }; -} - -process.on('exit', () => { - exit(0); -}); - -async function exit(status) { - logger.info('Exiting with status: %d ...', status); - logger.info('Stopping Azure Service Bus resources...') - if (receiver) { - try { - await receiver.close(); - } catch (e) { - - } - } - - if (receiverClient) { - try { - await receiverClient.close(); - } catch (e) { - - } - } - - senderMap.forEach((k, v) => { - try { - v.sender.close(); - } catch (e) { - - } - try { - v.queueClient.close(); - } catch (e) { - - } - }); - - if (sbClient) { - try { - sbClient.close(); - } catch (e) { - - } - } - logger.info('Azure Service Bus resources stopped.') - process.exit(status); -} \ No newline at end of file diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 58361016c4..f56e5bb766 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -13,38 +13,89 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +const { logLevel, Kafka } = require('kafkajs'); -const config = require('config'), logger = require('./config/logger')._logger('main'); - -const serviceType = config.get('service-type'); -switch (serviceType) { - case 'kafka': - logger.info('Starting kafka template.'); - require('./queue/kafkaTemplate'); - logger.info('kafka template started.'); - break; - case 'pubsub': - logger.info('Starting Pub/Sub template.') - require('./queue/pubSubTemplate'); - logger.info('Pub/Sub template started.') - break; - case 'aws-sqs': - logger.info('Starting Aws Sqs template.') - require('./queue/awsSqsTemplate'); - logger.info('Aws Sqs template started.') - break; - case 'rabbitmq': - logger.info('Starting RabbitMq template.') - require('./queue/rabbitmqTemplate'); - logger.info('RabbitMq template started.') - break; - case 'service-bus': - logger.info('Starting Azure Service Bus template.') - require('./queue/serviceBusTemplate'); - logger.info('Azure Service Bus template started.') - break; - default: - logger.error('Unknown service type: ', serviceType); - process.exit(-1); +const config = require('config'), + JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'), + logger = require('./config/logger')._logger('main'), + KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator; + +var kafkaClient; +var consumer; +var producer; + +(async() => { + try { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + + const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); + const kafkaRequestTopic = config.get('kafka.request_topic'); + + logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); + logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); + + kafkaClient = new Kafka({ + brokers: kafkaBootstrapServers.split(','), + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator + }); + + consumer = kafkaClient.consumer({ groupId: 'js-executor-group' }); + producer = kafkaClient.producer(); + const messageProcessor = new JsInvokeMessageProcessor(producer); + await consumer.connect(); + await producer.connect(); + await consumer.subscribe({ topic: kafkaRequestTopic}); + + logger.info('Started ThingsBoard JavaScript Executor Microservice.'); + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + messageProcessor.onJsInvokeMessage(message); + }, + }); + + } catch (e) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + exit(-1); + } +})(); + +process.on('exit', () => { + exit(0); +}); + +async function exit(status) { + logger.info('Exiting with status: %d ...', status); + if (consumer) { + logger.info('Stopping Kafka Consumer...'); + var _consumer = consumer; + consumer = null; + try { + await _consumer.disconnect(); + logger.info('Kafka Consumer stopped.'); + await disconnectProducer(); + process.exit(status); + } catch (e) { + logger.info('Kafka Consumer stop error.'); + await disconnectProducer(); + process.exit(status); + } + } else { + process.exit(status); + } } +async function disconnectProducer() { + if (producer) { + logger.info('Stopping Kafka Producer...'); + var _producer = producer; + producer = null; + try { + await _producer.disconnect(); + logger.info('Kafka Producer stopped.'); + } catch (e) { + logger.info('Kafka Producer stop error.'); + } + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java index e2bb07fb6f..1807c5c4fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/delay/TbMsgDelayNode.java @@ -41,7 +41,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; name = "delay", configClazz = TbMsgDelayNodeConfiguration.class, nodeDescription = "Delays incoming message", - nodeDetails = "Delays messages for configurable period. Please note, this node acknowledges the message from the current queue (message will be removed from queue)", + nodeDetails = "Delays messages for configurable period.", icon = "pause", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeMsgDelayConfig" From 4079daabcd1e00e39594b419935776f53ee4fed3 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 30 Apr 2020 12:50:41 +0300 Subject: [PATCH 139/196] Timeouts for Remote JS Executors --- .../service/script/AbstractJsInvokeService.java | 16 ++++++++++++++++ .../script/AbstractNashornJsInvokeService.java | 9 ++------- .../service/script/RemoteJsInvokeService.java | 16 +++++++++++++--- .../engine/filter/TbCheckAlarmStatusNode.java | 2 +- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java index daa74d013f..a903698e3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java @@ -18,10 +18,13 @@ package org.thingsboard.server.service.script; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; /** @@ -30,9 +33,22 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j public abstract class AbstractJsInvokeService implements JsInvokeService { + protected ScheduledExecutorService timeoutExecutorService; protected Map scriptIdToNameMap = new ConcurrentHashMap<>(); protected Map blackListedFunctions = new ConcurrentHashMap<>(); + public void init(long maxRequestsTimeout) { + if (maxRequestsTimeout > 0) { + timeoutExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("nashorn-js-timeout")); + } + } + + public void stop() { + if (timeoutExecutorService != null) { + timeoutExecutorService.shutdownNow(); + } + } + @Override public ListenableFuture eval(JsScriptType scriptType, String scriptBody, String... argNames) { UUID scriptId = UUID.randomUUID(); diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index a9acfa4f42..5ea8d13270 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -48,7 +48,6 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer private NashornSandbox sandbox; private ScriptEngine engine; private ExecutorService monitorExecutorService; - private ScheduledExecutorService timeoutExecutorService; private final AtomicInteger jsPushedMsgs = new AtomicInteger(0); private final AtomicInteger jsInvokeMsgs = new AtomicInteger(0); @@ -85,9 +84,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @PostConstruct public void init() { - if (maxRequestsTimeout > 0) { - timeoutExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("nashorn-js-timeout")); - } + super.init(maxRequestsTimeout); if (useJsSandbox()) { sandbox = NashornSandboxes.create(); monitorExecutorService = Executors.newWorkStealingPool(getMonitorThreadPoolSize()); @@ -104,12 +101,10 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @PreDestroy public void stop() { + super.stop(); if (monitorExecutorService != null) { monitorExecutorService.shutdownNow(); } - if (timeoutExecutorService != null) { - timeoutExecutorService.shutdownNow(); - } } protected abstract boolean useJsSandbox(); diff --git a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index fe2e7e8071..663cdc978d 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -87,11 +87,13 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @PostConstruct public void init() { + super.init(maxRequestsTimeout); requestTemplate.init(); } @PreDestroy public void destroy() { + super.stop(); if (requestTemplate != null) { requestTemplate.stop(); } @@ -111,7 +113,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { log.trace("Post compile request for scriptId [{}]", scriptId); ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); - + if (maxRequestsTimeout > 0) { + future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService); + } kafkaPushedMsgs.incrementAndGet(); Futures.addCallback(future, new FutureCallback>() { @Override @@ -154,8 +158,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .setTimeout((int) maxRequestsTimeout) .setScriptBody(scriptIdToBodysMap.get(scriptId)); - for (int i = 0; i < args.length; i++) { - jsRequestBuilder.addArgs(args[i].toString()); + for (Object arg : args) { + jsRequestBuilder.addArgs(arg.toString()); } JsInvokeProtos.RemoteJsRequest jsRequestWrapper = JsInvokeProtos.RemoteJsRequest.newBuilder() @@ -163,6 +167,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + if (maxRequestsTimeout > 0) { + future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService); + } kafkaPushedMsgs.incrementAndGet(); Futures.addCallback(future, new FutureCallback>() { @Override @@ -203,6 +210,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { .build(); ListenableFuture> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper)); + if (maxRequestsTimeout > 0) { + future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService); + } JsInvokeProtos.RemoteJsResponse response = future.get().getValue(); JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java index dc0eb9f396..7338270c9b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckAlarmStatusNode.java @@ -38,7 +38,7 @@ import java.io.IOException; @Slf4j @RuleNode( type = ComponentType.FILTER, - name = "checks alarm status", + name = "check alarm status", configClazz = TbCheckAlarmStatusNodeConfig.class, relationTypes = {"True", "False"}, nodeDescription = "Checks alarm status.", From 56ac2286625d87645437994d00ae22a15e002e05 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Thu, 30 Apr 2020 13:12:35 +0300 Subject: [PATCH 140/196] [WIP] Improvements for PostgreSQL 11 partitioning. (#2681) * improvements for migration to postgres 11 * tests update * fix typo * throw exception if query execution failed --- .../upgrade/2.4.3/schema_update_psql_ts.sql | 58 +++++++++++++++---- .../AbstractSqlTsDatabaseUpgradeService.java | 3 +- .../install/PsqlTsDatabaseSchemaService.java | 16 ++++- .../install/PsqlTsDatabaseUpgradeService.java | 17 +++++- .../SqlAbstractDatabaseSchemaService.java | 11 ++++ .../TimescaleTsDatabaseSchemaService.java | 19 ------ .../controller/ControllerSqlTestSuite.java | 2 +- .../server/mqtt/MqttSqlTestSuite.java | 2 +- .../server/rules/RuleEngineSqlTestSuite.java | 2 +- .../server/system/SystemSqlTestSuite.java | 2 +- ...stractChunkedAggregationTimeseriesDao.java | 2 +- .../dao/sqlts/hsql/JpaHsqlTimeseriesDao.java | 3 +- .../dao/sqlts/insert/InsertTsRepository.java | 3 +- .../insert/hsql/HsqlInsertTsRepository.java | 6 +- .../insert/psql/PsqlInsertTsRepository.java | 26 ++------- .../TimescaleInsertTsRepository.java | 5 +- .../dao/sqlts/psql/JpaPsqlTimeseriesDao.java | 49 ++++++---------- .../timescale/TimescaleTimeseriesDao.java | 5 +- .../server/dao/timeseries/PsqlPartition.java | 2 +- .../dao/timeseries/SqlTsPartitionDate.java | 10 +--- dao/src/main/resources/sql/schema-ts-psql.sql | 3 +- .../server/dao/JpaDaoTestSuite.java | 4 +- .../server/dao/SqlDaoServiceTestSuite.java | 4 +- .../sql/{ => hsql}/drop-all-tables.sql | 0 .../resources/sql/psql/drop-all-tables.sql | 24 ++++++++ .../sql/timescale/drop-all-tables.sql | 4 +- 26 files changed, 160 insertions(+), 122 deletions(-) rename dao/src/test/resources/sql/{ => hsql}/drop-all-tables.sql (100%) create mode 100644 dao/src/test/resources/sql/psql/drop-all-tables.sql diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql index e10f3ee2bd..671d39aae5 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql @@ -21,6 +21,8 @@ CREATE OR REPLACE PROCEDURE create_partition_ts_kv_table() LANGUAGE plpgsql AS $ BEGIN ALTER TABLE ts_kv RENAME TO ts_kv_old; + ALTER TABLE ts_kv_old + RENAME CONSTRAINT ts_kv_pkey TO ts_kv_pkey_old; CREATE TABLE IF NOT EXISTS ts_kv ( LIKE ts_kv_old @@ -32,6 +34,8 @@ BEGIN ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; ALTER TABLE ts_kv ALTER COLUMN key TYPE integer USING key::integer; + ALTER TABLE ts_kv + ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts); END; $$; @@ -59,33 +63,65 @@ BEGIN END; $$; +CREATE OR REPLACE FUNCTION get_partitions_data(IN partition_type varchar) + RETURNS + TABLE + ( + partition_date text, + from_ts bigint, + to_ts bigint + ) +AS +$$ +BEGIN + CASE + WHEN partition_type = 'DAYS' THEN + RETURN QUERY SELECT day_date.day AS partition_date, + (extract(epoch from (day_date.day)::timestamp) * 1000)::bigint AS from_ts, + (extract(epoch from (day_date.day::date + INTERVAL '1 DAY')::timestamp) * + 1000)::bigint AS to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_DD') AS day + FROM ts_kv_old) AS day_date; + WHEN partition_type = 'MONTHS' THEN + RETURN QUERY SELECT SUBSTRING(month_date.first_date, 1, 7) AS partition_date, + (extract(epoch from (month_date.first_date)::timestamp) * 1000)::bigint AS from_ts, + (extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) * + 1000)::bigint AS to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date + FROM ts_kv_old) AS month_date; + WHEN partition_type = 'YEARS' THEN + RETURN QUERY SELECT SUBSTRING(year_date.year, 1, 4) AS partition_date, + (extract(epoch from (year_date.year)::timestamp) * 1000)::bigint AS from_ts, + (extract(epoch from (year_date.year::date + INTERVAL '1 YEAR')::timestamp) * + 1000)::bigint AS to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_01_01') AS year FROM ts_kv_old) AS year_date; + ELSE + RAISE EXCEPTION 'Failed to parse partitioning property: % !', partition_type; + END CASE; +END; +$$ LANGUAGE plpgsql; -- call create_partitions(); -CREATE OR REPLACE PROCEDURE create_partitions() LANGUAGE plpgsql AS $$ +CREATE OR REPLACE PROCEDURE create_partitions(IN partition_type varchar) LANGUAGE plpgsql AS $$ DECLARE partition_date varchar; from_ts bigint; to_ts bigint; - key_cursor CURSOR FOR select SUBSTRING(month_date.first_date, 1, 7) AS partition_date, - extract(epoch from (month_date.first_date)::timestamp) * 1000 as from_ts, - extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) * - 1000 as to_ts - FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date - FROM ts_kv_old) AS month_date; + partitions_cursor CURSOR FOR SELECT * FROM get_partitions_data(partition_type); BEGIN - OPEN key_cursor; + OPEN partitions_cursor; LOOP - FETCH key_cursor INTO partition_date, from_ts, to_ts; + FETCH partitions_cursor INTO partition_date, from_ts, to_ts; EXIT WHEN NOT FOUND; EXECUTE 'CREATE TABLE IF NOT EXISTS ts_kv_' || partition_date || - ' PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (' || from_ts || + ' PARTITION OF ts_kv FOR VALUES FROM (' || from_ts || ') TO (' || to_ts || ');'; RAISE NOTICE 'A partition % has been created!',CONCAT('ts_kv_', partition_date); END LOOP; - CLOSE key_cursor; + CLOSE partitions_cursor; END; $$; diff --git a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java index ab03ca23ab..7afa422460 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java @@ -106,7 +106,8 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { Thread.sleep(2000); log.info("Successfully executed query: {}", query); } catch (InterruptedException | SQLException e) { - log.info("Failed to execute query: {} due to: {}", query, e.getMessage()); + log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); + throw new RuntimeException("Failed to execute query:" + query + " due to: ", e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java index 15b1e45247..bcc3d9bb81 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.install; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; @@ -24,9 +25,20 @@ import org.thingsboard.server.dao.util.SqlTsDao; @SqlTsDao @PsqlDao @Profile("install") -public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService - implements TsDatabaseSchemaService { +public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { + + @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") + private String partitionType; + public PsqlTsDatabaseSchemaService() { super("schema-ts-psql.sql", null); } + + @Override + public void createDatabaseSchema() throws Exception { + super.createDatabaseSchema(); + if (partitionType.equals("INDEFINITE")) { + executeQuery("CREATE TABLE ts_kv_indefinite PARTITION OF ts_kv DEFAULT;"); + } + } } \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index 5f97a6eaa5..215eff736a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.install; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; @@ -33,6 +34,9 @@ import java.sql.DriverManager; @PsqlDao public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { + @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") + private String partitionType; + private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql"; @@ -50,7 +54,6 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE; private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE; - private static final String CALL_CREATE_PARTITIONS = CALL_REGEX + CREATE_PARTITIONS; private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV; @@ -66,6 +69,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY; private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; + private static final String DROP_FUNCTION_GET_PARTITION_DATA = "DROP FUNCTION IF EXISTS get_partitions_data;"; @Override public void upgradeDatabase(String fromVersion) throws Exception { @@ -83,7 +87,11 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe loadSql(conn, LOAD_FUNCTIONS_SQL); log.info("Updating timeseries schema ..."); executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE); - executeQuery(conn, CALL_CREATE_PARTITIONS); + if (!partitionType.equals("INDEFINITE")) { + executeQuery(conn, "call create_partitions('" + partitionType + "')"); + } else { + executeQuery(conn, "CREATE TABLE IF NOT EXISTS ts_kv_indefinite PARTITION OF ts_kv DEFAULT;"); + } executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); executeQuery(conn, CALL_INSERT_INTO_TS_KV); @@ -100,9 +108,14 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV); executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); + executeQuery(conn, DROP_FUNCTION_GET_PARTITION_DATA); executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;"); + } else { + executeQuery(conn, "ALTER TABLE ts_kv DROP CONSTRAINT IF EXISTS ts_kv_pkey;"); + executeQuery(conn, "ALTER TABLE ts_kv ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts);"); } log.info("Load TTL functions ..."); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java index b8c0d964e1..cfc32d31b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; @Slf4j public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchemaService { @@ -73,4 +74,14 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema } } + protected void executeQuery(String query) { + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + log.info("Successfully executed query: {}", query); + Thread.sleep(5000); + } catch (InterruptedException | SQLException e) { + log.info("Failed to execute query: {} due to: {}", query, e.getMessage()); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java index e8c542d956..e2633d6fc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java @@ -33,14 +33,6 @@ import java.sql.SQLException; @Slf4j public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { - private static final String QUERY = "query: {}"; - private static final String SUCCESSFULLY_EXECUTED = "Successfully executed "; - private static final String FAILED_TO_EXECUTE = "Failed to execute "; - private static final String FAILED_DUE_TO = " due to: {}"; - - private static final String SUCCESSFULLY_EXECUTED_QUERY = SUCCESSFULLY_EXECUTED + QUERY; - private static final String FAILED_TO_EXECUTE_QUERY = FAILED_TO_EXECUTE + QUERY + FAILED_DUE_TO; - @Value("${sql.timescale.chunk_time_interval:86400000}") private long chunkTimeInterval; @@ -54,15 +46,4 @@ public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaS executeQuery("SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); } - private void executeQuery(String query) { - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - log.info(SUCCESSFULLY_EXECUTED_QUERY, query); - Thread.sleep(5000); - } catch (InterruptedException | SQLException e) { - log.info(FAILED_TO_EXECUTE_QUERY, query, e.getMessage()); - } - } - - } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index 347eaee7eb..d8653e945d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -33,7 +33,7 @@ public class ControllerSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), - "sql/drop-all-tables.sql", + "sql/hsql/drop-all-tables.sql", "sql-test.properties"); @BeforeClass diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 70de3b27ca..69a6e99353 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -32,7 +32,7 @@ public class MqttSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), - "sql/drop-all-tables.sql", + "sql/hsql/drop-all-tables.sql", "sql-test.properties"); @BeforeClass diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index 83ac5f7703..72484e3f21 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -33,7 +33,7 @@ public class RuleEngineSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), - "sql/drop-all-tables.sql", + "sql/hsql/drop-all-tables.sql", "sql-test.properties"); @BeforeClass diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index 6060a2cc2b..36d24e9b00 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -34,7 +34,7 @@ public class SystemSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), - "sql/drop-all-tables.sql", + "sql/hsql/drop-all-tables.sql", "sql-test.properties"); @BeforeClass diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index fafe05957f..c663d4346a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -54,7 +54,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @Autowired protected InsertTsRepository insertRepository; - protected TbSqlBlockingQueue> tsQueue; + protected TbSqlBlockingQueue tsQueue; @PostConstruct protected void init() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java index 41ecfc3734..5d94d12a1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sqlts.AbstractChunkedAggregationTimeseriesDao; -import org.thingsboard.server.dao.sqlts.EntityContainer; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.dao.util.HsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; @@ -48,7 +47,7 @@ public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(new EntityContainer(entity, null)); + return tsQueue.add(entity); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/InsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/InsertTsRepository.java index a2c066322a..9618d79230 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/InsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/InsertTsRepository.java @@ -16,12 +16,11 @@ package org.thingsboard.server.dao.sqlts.insert; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; -import org.thingsboard.server.dao.sqlts.EntityContainer; import java.util.List; public interface InsertTsRepository { - void saveOrUpdate(List> entities); + void saveOrUpdate(List entities); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java index 189758a947..5fd585aec8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/hsql/HsqlInsertTsRepository.java @@ -19,7 +19,6 @@ import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sqlts.EntityContainer; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.util.HsqlDao; @@ -47,12 +46,11 @@ public class HsqlInsertTsRepository extends AbstractInsertRepository implements "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v, T.json_v);"; @Override - public void saveOrUpdate(List> entities) { + public void saveOrUpdate(List entities) { jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { - EntityContainer tsKvEntityEntityContainer = entities.get(i); - TsKvEntity tsKvEntity = tsKvEntityEntityContainer.getEntity(); + TsKvEntity tsKvEntity = entities.get(i); ps.setObject(1, tsKvEntity.getEntityId()); ps.setInt(2, tsKvEntity.getKey()); ps.setLong(3, tsKvEntity.getTs()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java index d4a5dd25b0..85028bb942 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/psql/PsqlInsertTsRepository.java @@ -20,7 +20,6 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; -import org.thingsboard.server.dao.sqlts.EntityContainer; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; @@ -28,10 +27,7 @@ import org.thingsboard.server.dao.util.SqlTsDao; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; @SqlTsDao @PsqlDao @@ -39,22 +35,15 @@ import java.util.Map; @Transactional public class PsqlInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository { - private static final String INSERT_INTO_TS_KV = "INSERT INTO ts_kv_"; - - private static final String VALUES_ON_CONFLICT_DO_UPDATE = " (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES (?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " + + private static final String INSERT_ON_CONFLICT_DO_UPDATE = "INSERT INTO ts_kv (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES (?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " + "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);"; @Override - public void saveOrUpdate(List> entities) { - Map> partitionMap = new HashMap<>(); - for (EntityContainer entityContainer : entities) { - List tsKvEntities = partitionMap.computeIfAbsent(entityContainer.getPartitionDate(), k -> new ArrayList<>()); - tsKvEntities.add(entityContainer.getEntity()); - } - partitionMap.forEach((partition, entries) -> jdbcTemplate.batchUpdate(getInsertOrUpdateQuery(partition), new BatchPreparedStatementSetter() { + public void saveOrUpdate(List entities) { + jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE, new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvEntity tsKvEntity = entries.get(i); + TsKvEntity tsKvEntity = entities.get(i); ps.setObject(1, tsKvEntity.getEntityId()); ps.setInt(2, tsKvEntity.getKey()); ps.setLong(3, tsKvEntity.getTs()); @@ -93,12 +82,9 @@ public class PsqlInsertTsRepository extends AbstractInsertRepository implements @Override public int getBatchSize() { - return entries.size(); + return entities.size(); } - })); + }); } - private String getInsertOrUpdateQuery(String partitionDate) { - return INSERT_INTO_TS_KV + partitionDate + VALUES_ON_CONFLICT_DO_UPDATE; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java index 1fa1fc4219..da91982f5d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java @@ -20,7 +20,6 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; -import org.thingsboard.server.dao.sqlts.EntityContainer; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; @@ -41,11 +40,11 @@ public class TimescaleInsertTsRepository extends AbstractInsertRepository implem "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);"; @Override - public void saveOrUpdate(List> entities) { + public void saveOrUpdate(List entities) { jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { - TimescaleTsKvEntity tsKvEntity = entities.get(i).getEntity(); + TimescaleTsKvEntity tsKvEntity = entities.get(i); ps.setObject(1, tsKvEntity.getEntityId()); ps.setInt(2, tsKvEntity.getKey()); ps.setLong(3, tsKvEntity.getTs()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java index 29cdd64918..e0854a0a19 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sqlts.AbstractChunkedAggregationTimeseriesDao; -import org.thingsboard.server.dao.sqlts.EntityContainer; import org.thingsboard.server.dao.sqlts.insert.psql.PsqlPartitioningRepository; import org.thingsboard.server.dao.timeseries.PsqlPartition; import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate; @@ -42,8 +41,6 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; -import static org.thingsboard.server.dao.timeseries.SqlTsPartitionDate.EPOCH_START; - @Component @Slf4j @@ -58,7 +55,6 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa private PsqlPartitioningRepository partitioningRepository; private SqlTsPartitionDate tsFormat; - private PsqlPartition indefinitePartition; @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") private String partitioning; @@ -69,10 +65,6 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa Optional partition = SqlTsPartitionDate.parse(partitioning); if (partition.isPresent()) { tsFormat = partition.get(); - if (tsFormat.equals(SqlTsPartitionDate.INDEFINITE)) { - indefinitePartition = new PsqlPartition(toMills(EPOCH_START), Long.MAX_VALUE, tsFormat.getPattern()); - savePartition(indefinitePartition); - } } else { log.warn("Incorrect configuration of partitioning {}", partitioning); throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!"); @@ -81,6 +73,7 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + savePartitionIfNotExist(tsKvEntry.getTs()); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TsKvEntity entity = new TsKvEntity(); @@ -92,9 +85,23 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - PsqlPartition psqlPartition = toPartition(tsKvEntry.getTs()); log.trace("Saving entity: {}", entity); - return tsQueue.add(new EntityContainer(entity, psqlPartition.getPartitionDate())); + return tsQueue.add(entity); + } + + private void savePartitionIfNotExist(long ts) { + if (!tsFormat.equals(SqlTsPartitionDate.INDEFINITE)) { + LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); + LocalDateTime localDateTimeStart = tsFormat.trancateTo(time); + long partitionStartTs = toMills(localDateTimeStart); + if (partitions.get(partitionStartTs) == null) { + LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart); + long partitionEndTs = toMills(localDateTimeEnd); + ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC); + String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern())); + savePartition(new PsqlPartition(partitionStartTs, partitionEndTs, partitionDate)); + } + } } private void savePartition(PsqlPartition psqlPartition) { @@ -111,28 +118,6 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa } } - private PsqlPartition toPartition(long ts) { - if (tsFormat.equals(SqlTsPartitionDate.INDEFINITE)) { - return indefinitePartition; - } else { - LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); - LocalDateTime localDateTimeStart = tsFormat.trancateTo(time); - long partitionStartTs = toMills(localDateTimeStart); - PsqlPartition partition = partitions.get(partitionStartTs); - if (partition != null) { - return partition; - } else { - LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart); - long partitionEndTs = toMills(localDateTimeEnd); - ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC); - String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern())); - partition = new PsqlPartition(partitionStartTs, partitionEndTs, partitionDate); - savePartition(partition); - return partition; - } - } - } - private static long toMills(LocalDateTime time) { return time.toInstant(ZoneOffset.UTC).toEpochMilli(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 0a2e210648..d9121f684e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -36,7 +36,6 @@ import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; -import org.thingsboard.server.dao.sqlts.EntityContainer; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; @@ -64,7 +63,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Autowired protected InsertTsRepository insertRepository; - protected TbSqlBlockingQueue> tsQueue; + protected TbSqlBlockingQueue tsQueue; @PostConstruct protected void init() { @@ -175,7 +174,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity to timescale db: {}", entity); - return tsQueue.add(new EntityContainer(entity, null)); + return tsQueue.add(entity); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java index 09f5312498..ffb7dec97e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java @@ -35,6 +35,6 @@ public class PsqlPartition { } private String createStatement(long start, long end, String partitionDate) { - return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (" + start + ") TO (" + end + ")"; + return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv FOR VALUES FROM (" + start + ") TO (" + end + ")"; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java index ceede6f19c..7edbafd816 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java @@ -23,7 +23,7 @@ import java.util.Optional; public enum SqlTsPartitionDate { - MINUTES("yyyy_MM_dd_HH_mm", ChronoUnit.MINUTES), HOURS("yyyy_MM_dd_HH", ChronoUnit.HOURS), DAYS("yyyy_MM_dd", ChronoUnit.DAYS), MONTHS("yyyy_MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS), INDEFINITE("indefinite", ChronoUnit.FOREVER); + DAYS("yyyy_MM_dd", ChronoUnit.DAYS), MONTHS("yyyy_MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS), INDEFINITE("indefinite", ChronoUnit.FOREVER); private final String pattern; private final transient TemporalUnit truncateUnit; @@ -44,10 +44,6 @@ public enum SqlTsPartitionDate { public LocalDateTime trancateTo(LocalDateTime time) { switch (this) { - case MINUTES: - return time.truncatedTo(ChronoUnit.MINUTES); - case HOURS: - return time.truncatedTo(ChronoUnit.HOURS); case DAYS: return time.truncatedTo(ChronoUnit.DAYS); case MONTHS: @@ -63,10 +59,6 @@ public enum SqlTsPartitionDate { public LocalDateTime plusTo(LocalDateTime time) { switch (this) { - case MINUTES: - return time.plusMinutes(1); - case HOURS: - return time.plusHours(1); case DAYS: return time.plusDays(1); case MONTHS: diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index f83e80931b..6f6177de01 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -23,7 +23,8 @@ CREATE TABLE IF NOT EXISTS ts_kv str_v varchar(10000000), long_v bigint, dbl_v double precision, - json_v json + json_v json, + CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS ts_kv_latest diff --git a/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java index 0af8603bf1..23ec159a4b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java @@ -31,14 +31,14 @@ public class JpaDaoTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/system-data.sql"), - "sql/drop-all-tables.sql", + "sql/hsql/drop-all-tables.sql", "sql-test.properties" ); // @ClassRule // public static CustomSqlUnit sqlUnit = new CustomSqlUnit( // Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), -// "sql/drop-all-tables.sql", +// "sql/psql/drop-all-tables.sql", // "sql-test.properties" // ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java index 6d306c4e71..32fd45c188 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java @@ -31,14 +31,14 @@ public class SqlDaoServiceTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities-hsql.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), - "sql/drop-all-tables.sql", + "sql/hsql/drop-all-tables.sql", "sql-test.properties" ); // @ClassRule // public static CustomSqlUnit sqlUnit = new CustomSqlUnit( // Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), -// "sql/drop-all-tables.sql", +// "sql/psql/drop-all-tables.sql", // "sql-test.properties" // ); diff --git a/dao/src/test/resources/sql/drop-all-tables.sql b/dao/src/test/resources/sql/hsql/drop-all-tables.sql similarity index 100% rename from dao/src/test/resources/sql/drop-all-tables.sql rename to dao/src/test/resources/sql/hsql/drop-all-tables.sql diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql new file mode 100644 index 0000000000..b2e4a27963 --- /dev/null +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -0,0 +1,24 @@ +DROP TABLE IF EXISTS admin_settings; +DROP TABLE IF EXISTS alarm; +DROP TABLE IF EXISTS asset; +DROP TABLE IF EXISTS audit_log; +DROP TABLE IF EXISTS attribute_kv; +DROP TABLE IF EXISTS component_descriptor; +DROP TABLE IF EXISTS customer; +DROP TABLE IF EXISTS dashboard; +DROP TABLE IF EXISTS device; +DROP TABLE IF EXISTS device_credentials; +DROP TABLE IF EXISTS event; +DROP TABLE IF EXISTS relation; +DROP TABLE IF EXISTS tb_user; +DROP TABLE IF EXISTS tenant; +DROP TABLE IF EXISTS ts_kv; +DROP TABLE IF EXISTS ts_kv_latest; +DROP TABLE IF EXISTS ts_kv_dictionary; +DROP TABLE IF EXISTS user_credentials; +DROP TABLE IF EXISTS widget_type; +DROP TABLE IF EXISTS widgets_bundle; +DROP TABLE IF EXISTS rule_node; +DROP TABLE IF EXISTS rule_chain; +DROP TABLE IF EXISTS entity_view; +DROP TABLE IF EXISTS tb_schema_settings; \ No newline at end of file diff --git a/dao/src/test/resources/sql/timescale/drop-all-tables.sql b/dao/src/test/resources/sql/timescale/drop-all-tables.sql index ac921c0f4a..b2e4a27963 100644 --- a/dao/src/test/resources/sql/timescale/drop-all-tables.sql +++ b/dao/src/test/resources/sql/timescale/drop-all-tables.sql @@ -14,9 +14,11 @@ DROP TABLE IF EXISTS tb_user; DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; DROP TABLE IF EXISTS ts_kv_latest; +DROP TABLE IF EXISTS ts_kv_dictionary; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widget_type; DROP TABLE IF EXISTS widgets_bundle; DROP TABLE IF EXISTS rule_node; DROP TABLE IF EXISTS rule_chain; -DROP TABLE IF EXISTS entity_view; \ No newline at end of file +DROP TABLE IF EXISTS entity_view; +DROP TABLE IF EXISTS tb_schema_settings; \ No newline at end of file From 30a2d19d2dca8f0df5b4a9c949692703e2ac3346 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 30 Apr 2020 15:17:39 +0300 Subject: [PATCH 141/196] Fixes to Kafka JS headers issue --- msa/js-executor/api/jsInvokeMessageProcessor.js | 2 +- msa/js-executor/queue/kafkaTemplate.js | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 79ea505bcf..a4398424db 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -59,7 +59,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { } else if (request.releaseRequest) { this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest); } else { - logger.error('[%s] Unknown request recevied!', requestId); + logger.error('[%s] Unknown request received!', requestId); } } catch (err) { diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 699ca7be00..6a09338427 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -41,8 +41,6 @@ function KafkaProducer() { } } - let headersData = headers.data; - headersData = Object.fromEntries(Object.entries(headersData).map(([key, value]) => [key, Buffer.from(value)])); return producer.send( { topic: responseTopic, @@ -50,7 +48,7 @@ function KafkaProducer() { { key: scriptId, value: rawResponse, - headers: headersData + headers: headers.data } ] }); @@ -96,15 +94,10 @@ function KafkaProducer() { eachMessage: async ({topic, partition, message}) => { let headers = message.headers; let key = message.key; - let data = message.value; let msg = {}; - - headers = Object.fromEntries( - Object.entries(headers).map(([key, value]) => [key, [...value]])); - msg.key = key.toString('utf8'); - msg.data = [...data]; - msg.headers = {data: headers} + msg.data = message.value; + msg.headers = {data: headers}; messageProcessor.onJsInvokeMessage(msg); }, }); From e39ccaa2ccf6817488038111a9b72eb3007af76c Mon Sep 17 00:00:00 2001 From: VoBa Date: Thu, 30 Apr 2020 15:48:07 +0300 Subject: [PATCH 142/196] [2.5] Improved login by params to support nagivation to different dashboards (#2702) * Improved login by params * Fix * Removed incorrect redirect * Refactoring --- ui/src/app/api/user.service.js | 20 ++++++++++++++++++++ ui/src/app/login/login.controller.js | 8 +------- ui/src/app/login/login.routes.js | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ui/src/app/api/user.service.js b/ui/src/app/api/user.service.js index dacfe955a2..9ac8270403 100644 --- a/ui/src/app/api/user.service.js +++ b/ui/src/app/api/user.service.js @@ -386,6 +386,26 @@ function UserService($http, $q, $rootScope, adminService, dashboardService, time deferred.reject(); } procceedJwtTokenValidate(); + } else if (locationSearch.username && locationSearch.password) { + var user = {}; + user.name = locationSearch.username; + user.password = locationSearch.password; + $location.search('username', null); + $location.search('password', null); + + loginService.login(user).then(function success(response) { + var token = response.data.token; + var refreshToken = response.data.refreshToken; + try { + updateAndValidateToken(token, 'jwt_token', false); + updateAndValidateToken(refreshToken, 'refresh_token', false); + } catch (e) { + deferred.reject(); + } + procceedJwtTokenValidate(); + }, function fail() { + deferred.reject(); + }); } else { procceedJwtTokenValidate(); } diff --git a/ui/src/app/login/login.controller.js b/ui/src/app/login/login.controller.js index d749ea0936..8dcd2a9d8e 100644 --- a/ui/src/app/login/login.controller.js +++ b/ui/src/app/login/login.controller.js @@ -20,7 +20,7 @@ import logoSvg from '../../svg/logo_title_white.svg'; /* eslint-enable import/no-unresolved, import/default */ /*@ngInject*/ -export default function LoginController(toast, loginService, userService, types, $state, $stateParams/*, $rootScope, $log, $translate*/) { +export default function LoginController(toast, loginService, userService, types, $state/*, $rootScope, $log, $translate*/) { var vm = this; vm.logoSvg = logoSvg; @@ -32,12 +32,6 @@ export default function LoginController(toast, loginService, userService, types, vm.login = login; - if ($stateParams.username && $stateParams.password) { - vm.user.name = $stateParams.username; - vm.user.password = $stateParams.password; - doLogin(); - } - function doLogin() { loginService.login(vm.user).then(function success(response) { var token = response.data.token; diff --git a/ui/src/app/login/login.routes.js b/ui/src/app/login/login.routes.js index 51121a782a..ac9401147b 100644 --- a/ui/src/app/login/login.routes.js +++ b/ui/src/app/login/login.routes.js @@ -25,7 +25,7 @@ import createPasswordTemplate from './create-password.tpl.html'; /*@ngInject*/ export default function LoginRoutes($stateProvider) { $stateProvider.state('login', { - url: '/login?username&password', + url: '/login', module: 'public', views: { "@": { From 9ef3445b7723a48ad2b96d6cf3d1b3830e946635 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Apr 2020 16:24:12 +0300 Subject: [PATCH 143/196] refactored --- .../config/custom-environment-variables.yml | 12 ++++++------ msa/js-executor/config/default.yml | 12 ++++++------ msa/js-executor/queue/awsSqsTemplate.js | 2 +- msa/js-executor/queue/kafkaTemplate.js | 2 +- msa/js-executor/queue/pubSubTemplate.js | 2 +- msa/js-executor/queue/rabbitmqTemplate.js | 2 +- msa/js-executor/queue/serviceBusTemplate.js | 2 +- msa/js-executor/server.js | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b290719739..c573274801 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) +queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" js: @@ -25,18 +25,18 @@ kafka: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" - topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" + topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" pubsub: project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" - queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" aws_sqs: access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" region: "TB_QUEUE_AWS_SQS_REGION" - queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" rabbitmq: host: "TB_QUEUE_RABBIT_MQ_HOST" @@ -44,14 +44,14 @@ rabbitmq: virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" username: "TB_QUEUE_RABBIT_MQ_USERNAME" password: "TB_QUEUE_RABBIT_MQ_PASSWORD" - queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" service_bus: namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" - queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" logger: level: "LOGGER_LEVEL" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 3155b051dc..f42b74745f 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "kafka" +queue_type: "kafka" request_topic: "js_eval.requests" js: @@ -25,13 +25,13 @@ kafka: # Kafka Bootstrap Servers servers: "localhost:9092" replication_factor: "1" - topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" pubsub: - queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" + queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" aws_sqs: - queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" + queue_properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" rabbitmq: host: "localhost" @@ -39,10 +39,10 @@ rabbitmq: virtual_host: "/" username: "admin" password: "password" - queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" + queue_properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" service_bus: - queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" + queue_properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" logger: level: "info" diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index 0396824af1..a5338c5e73 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -26,7 +26,7 @@ const accessKeyId = config.get('aws_sqs.access_key_id'); const secretAccessKey = config.get('aws_sqs.secret_access_key'); const region = config.get('aws_sqs.region'); const AWS = require('aws-sdk'); -const queueProperties = config.get('aws_sqs.queue-properties'); +const queueProperties = config.get('aws_sqs.queue_properties'); const poolInterval = config.get('js.response_poll_interval'); let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 699ca7be00..e22ec8cd71 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -20,7 +20,7 @@ const config = require('config'), logger = require('../config/logger')._logger('kafkaTemplate'), KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; const replicationFactor = config.get('kafka.replication_factor'); -const topicProperties = config.get('kafka.topic-properties'); +const topicProperties = config.get('kafka.topic_properties'); let kafkaClient; let kafkaAdmin; diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 17e1b56e1d..7d0b32ea34 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -24,7 +24,7 @@ const {PubSub} = require('@google-cloud/pubsub'); const projectId = config.get('pubsub.project_id'); const credentials = JSON.parse(config.get('pubsub.service_account')); const requestTopic = config.get('request_topic'); -const queueProperties = config.get('pubsub.queue-properties'); +const queueProperties = config.get('pubsub.queue_properties'); let pubSubClient; diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js index 1a2905c3a0..732206ff11 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.js +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -26,7 +26,7 @@ const port = config.get('rabbitmq.port'); const vhost = config.get('rabbitmq.virtual_host'); const username = config.get('rabbitmq.username'); const password = config.get('rabbitmq.password'); -const queueProperties = config.get('rabbitmq.queue-properties'); +const queueProperties = config.get('rabbitmq.queue_properties'); const poolInterval = config.get('js.response_poll_interval'); const amqp = require('amqplib/callback_api'); diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js index 034921afb7..20cf664940 100644 --- a/msa/js-executor/queue/serviceBusTemplate.js +++ b/msa/js-executor/queue/serviceBusTemplate.js @@ -26,7 +26,7 @@ const requestTopic = config.get('request_topic'); const namespaceName = config.get('service_bus.namespace_name'); const sasKeyName = config.get('service_bus.sas_key_name'); const sasKey = config.get('service_bus.sas_key'); -const queueProperties = config.get('service_bus.queue-properties'); +const queueProperties = config.get('service_bus.queue_properties'); let sbClient; let receiverClient; diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 58361016c4..e57ba62c11 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -16,7 +16,7 @@ const config = require('config'), logger = require('./config/logger')._logger('main'); -const serviceType = config.get('service-type'); +const serviceType = config.get('queue_type'); switch (serviceType) { case 'kafka': logger.info('Starting kafka template.'); From a437eba5857436a38e76bddfc73d508b5755b2a8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 30 Apr 2020 17:29:06 +0300 Subject: [PATCH 144/196] Fixed default Rule Chain --- .../src/main/data/json/demo/rule_chains/root_rule_chain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 865d8cb8e6..32cecf6e4e 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -3,7 +3,7 @@ "additionalInfo": null, "name": "Root Rule Chain", "firstRuleNodeId": null, - "root": false, + "root": true, "debugMode": false, "configuration": null }, From 702a8b6139e2a2ea16c42f4301b9936fabc82e5b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 30 Apr 2020 18:10:05 +0300 Subject: [PATCH 145/196] Force unsubscribe from Kafka topics --- .../thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java | 1 + docker/tb-node.env | 2 ++ 2 files changed, 3 insertions(+) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 49d1d74102..541337579d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -106,6 +106,7 @@ public class TbKafkaConsumerTemplate implements TbQueueCon if (!subscribed) { List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); topicNames.forEach(admin::createTopicIfNotExists); + consumer.unsubscribe(); consumer.subscribe(topicNames); subscribed = true; } diff --git a/docker/tb-node.env b/docker/tb-node.env index 542296babe..12cdc7d035 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -10,3 +10,5 @@ CACHE_TYPE=redis REDIS_HOST=redis HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false + +TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE=64 From 89d19fb36893bfff7a0722beaf597adb819803d9 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Thu, 30 Apr 2020 18:40:45 +0300 Subject: [PATCH 146/196] fix typo --- .../server/service/install/PsqlTsDatabaseUpgradeService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index 215eff736a..a06ef0fe05 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -46,7 +46,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe private static final String CREATE_PARTITION_TS_KV_TABLE = "create_partition_ts_kv_table()"; private static final String CREATE_NEW_TS_KV_LATEST_TABLE = "create_new_ts_kv_latest_table()"; - private static final String CREATE_PARTITIONS = "create_partitions()"; + private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)"; private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; @@ -108,7 +108,6 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV); executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); executeQuery(conn, DROP_FUNCTION_GET_PARTITION_DATA); executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); From e5c5aa705fa308397536c18c0779480d849ab368 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Apr 2020 20:14:34 +0300 Subject: [PATCH 147/196] refactored --- docker/tb-js-executor.env | 2 +- msa/js-executor/queue/awsSqsTemplate.js | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index 0b64f43b00..b66073ea44 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,4 +1,4 @@ - +TB_QUEUE_TYPE=kafka REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index a5338c5e73..e0ffb55c87 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -74,11 +74,13 @@ function AwsSqsProducer() { const queues = await getQueues(); - queues.forEach(queueUrl => { - const delimiterPosition = queueUrl.lastIndexOf('/'); - const queueName = queueUrl.substring(delimiterPosition + 1); - queueUrls.set(queueName, queueUrl); - }) + if (queues) { + queues.forEach(queueUrl => { + const delimiterPosition = queueUrl.lastIndexOf('/'); + const queueName = queueUrl.substring(delimiterPosition + 1); + queueUrls.set(queueName, queueUrl); + }); + } parseQueueProperties(); From 09d8823205e5d517cec66ff3bd4a84e9d491ee60 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 30 Apr 2020 20:38:53 +0300 Subject: [PATCH 148/196] Race condition fix --- .../queue/kafka/TbKafkaConsumerTemplate.java | 26 ++++++++++++------- docker/docker-compose.yml | 2 +- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 541337579d..6b1c051eeb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -81,14 +81,24 @@ public class TbKafkaConsumerTemplate implements TbQueueCon @Override public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + consumerLock.lock(); + try { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } finally { + consumerLock.unlock(); + } } @Override public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + consumerLock.lock(); + try { + this.partitions = partitions; + subscribed = false; + } finally { + consumerLock.unlock(); + } } @Override @@ -100,13 +110,11 @@ public class TbKafkaConsumerTemplate implements TbQueueCon log.debug("Failed to await subscription", e); } } else { + consumerLock.lock(); try { - consumerLock.lock(); - if (!subscribed) { List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); topicNames.forEach(admin::createTopicIfNotExists); - consumer.unsubscribe(); consumer.subscribe(topicNames); subscribed = true; } @@ -132,8 +140,8 @@ public class TbKafkaConsumerTemplate implements TbQueueCon @Override public void commit() { + consumerLock.lock(); try { - consumerLock.lock(); consumer.commitAsync(); } finally { consumerLock.unlock(); @@ -142,8 +150,8 @@ public class TbKafkaConsumerTemplate implements TbQueueCon @Override public void unsubscribe() { + consumerLock.lock(); try { - consumerLock.lock(); if (consumer != null) { consumer.unsubscribe(); consumer.close(); diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2f0d0fb3b3..9061f3e2de 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,7 +28,7 @@ services: ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 kafka: restart: always - image: "wurstmeister/kafka:2.12-2.2.1" + image: "wurstmeister/kafka:2.12-2.3.0" ports: - "9092:9092" env_file: From c7f282d39385473928f4111d015bc02ce30b07bb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 1 May 2020 12:45:06 +0300 Subject: [PATCH 149/196] Refactoring of the Queue Consumers --- .../AbstractTbQueueConsumerTemplate.java | 136 ++++++++++++++++++ .../queue/kafka/TbKafkaConsumerTemplate.java | 106 +++----------- 2 files changed, 159 insertions(+), 83 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java new file mode 100644 index 0000000000..084d10fca9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Slf4j +public abstract class AbstractTbQueueConsumerTemplate implements TbQueueConsumer { + + private volatile boolean subscribed; + protected volatile Set partitions; + protected final Lock consumerLock = new ReentrantLock(); + + @Getter + private final String topic; + + public AbstractTbQueueConsumerTemplate(String topic) { + this.topic = topic; + } + + @Override + public void subscribe() { + consumerLock.lock(); + try { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } finally { + consumerLock.unlock(); + } + } + + @Override + public void subscribe(Set partitions) { + consumerLock.lock(); + try { + this.partitions = partitions; + subscribed = false; + } finally { + consumerLock.unlock(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + consumerLock.lock(); + try { + if (!subscribed) { + doSubscribe(); + subscribed = true; + } + + List records = doPoll(durationInMillis); + if (!records.isEmpty()) { + List result = new ArrayList<>(records.size()); + records.forEach(record -> { + try { + if (record != null) { + result.add(decode(record)); + } + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + throw new RuntimeException("Failed to decode record: ", e); + } + }); + return result; + } + } finally { + consumerLock.unlock(); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + consumerLock.lock(); + try { + doCommit(); + } finally { + consumerLock.unlock(); + } + } + + @Override + public void unsubscribe() { + consumerLock.lock(); + try { + doUnsubscribe(); + } finally { + consumerLock.unlock(); + } + } + + abstract protected List doPoll(long durationInMillis); + + abstract protected T decode(R record) throws IOException; + + abstract protected void doSubscribe(); + + abstract protected void doCommit(); + + abstract protected void doUnsubscribe(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 6b1c051eeb..fea31854df 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -24,8 +23,8 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; @@ -33,26 +32,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. */ @Slf4j -public class TbKafkaConsumerTemplate implements TbQueueConsumer { +public class TbKafkaConsumerTemplate extends AbstractTbQueueConsumerTemplate, T> { private final TbQueueAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; - private volatile boolean subscribed; - private volatile Set partitions; - private final Lock consumerLock; - - @Getter - private final String topic; @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, @@ -60,6 +50,7 @@ public class TbKafkaConsumerTemplate implements TbQueueCon boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords, TbQueueAdmin admin) { + super(topic); Properties props = settings.toProps(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); if (groupId != null) { @@ -75,94 +66,43 @@ public class TbKafkaConsumerTemplate implements TbQueueCon this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; - this.topic = topic; - this.consumerLock = new ReentrantLock(); } @Override - public void subscribe() { - consumerLock.lock(); - try { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } finally { - consumerLock.unlock(); - } + protected void doSubscribe() { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); } @Override - public void subscribe(Set partitions) { - consumerLock.lock(); - try { - this.partitions = partitions; - subscribed = false; - } finally { - consumerLock.unlock(); + protected List> doPoll(long durationInMillis) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.isEmpty()) { + return Collections.emptyList(); + } else { + List> recordList = new ArrayList<>(256); + records.forEach(recordList::add); + return recordList; } } @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - consumerLock.lock(); - try { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; - } - } finally { - consumerLock.unlock(); - } - } - return Collections.emptyList(); + public T decode(ConsumerRecord record) throws IOException { + return decoder.decode(new KafkaTbQueueMsg(record)); } @Override - public void commit() { - consumerLock.lock(); - try { - consumer.commitAsync(); - } finally { - consumerLock.unlock(); - } + protected void doCommit() { + consumer.commitAsync(); } @Override - public void unsubscribe() { - consumerLock.lock(); - try { - if (consumer != null) { - consumer.unsubscribe(); - consumer.close(); - } - } finally { - consumerLock.unlock(); + protected void doUnsubscribe() { + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); } } - public T decode(ConsumerRecord record) throws IOException { - return decoder.decode(new KafkaTbQueueMsg(record)); - } - } From eedb38384536215555c97cb4fcff53ae7ee1bf72 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 1 May 2020 14:15:31 +0300 Subject: [PATCH 150/196] AWS improvements --- .../server/queue/sqs/TbAwsSqsConsumerTemplate.java | 6 +++++- .../server/queue/sqs/TbAwsSqsProducerTemplate.java | 5 ++++- .../server/queue/sqs/TbAwsSqsQueueAttributes.java | 1 - msa/js-executor/package.json | 1 + msa/js-executor/queue/awsSqsTemplate.js | 5 +++-- msa/js-executor/queue/pubSubTemplate.js | 8 +++++--- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 3e71388844..b66cad1504 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -127,6 +127,11 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo if (!subscribed) { List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + } + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); subscribed = true; } @@ -172,7 +177,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo ReceiveMessageRequest request = new ReceiveMessageRequest(); request .withWaitTimeSeconds(waitTimeSeconds) - .withMessageAttributeNames("headers") .withQueueUrl(url) .withMaxNumberOfMessages(MAX_NUM_MSGS); return sqsClient.receiveMessage(request).getMessages(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 2d85539184..6110d08c5e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -80,7 +81,9 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + sendMsgRequest.withMessageGroupId(tpi.getTopic()); + sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); Futures.addCallback(future, new FutureCallback() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 70a9587bef..c6cbbfd256 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -55,7 +55,6 @@ public class TbAwsSqsQueueAttributes { @PostConstruct private void init() { defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); coreAttributes = getConfigs(coreProperties); ruleEngineAttributes = getConfigs(ruleEngineProperties); diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 3dadac4f84..60a107061a 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -22,6 +22,7 @@ "azure-sb": "^0.11.1", "long": "^4.0.0", "uuid-parse": "^1.0.0", + "uuid-random": "^1.3.0", "winston": "^3.0.0", "winston-daily-rotate-file": "^3.2.1" }, diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index e0ffb55c87..5f95de7d32 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -19,6 +19,7 @@ const config = require('config'), JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), logger = require('../config/logger')._logger('awsSqsTemplate'); +const uuid = require('uuid-random'); const requestTopic = config.get('request_topic'); @@ -29,7 +30,7 @@ const AWS = require('aws-sdk'); const queueProperties = config.get('aws_sqs.queue_properties'); const poolInterval = config.get('js.response_poll_interval'); -let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; +let queueAttributes = {FifoQueue: 'true'}; let sqsClient; let requestQueueURL; const queueUrls = new Map(); @@ -51,7 +52,7 @@ function AwsSqsProducer() { queueUrls.set(responseTopic, responseQueueUrl); } - let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; + let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: 'js_eval', MessageDeduplicationId: uuid()}; return new Promise((resolve, reject) => { sqsClient.sendMessage(params, function (err, data) { diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 7d0b32ea34..cc5284022d 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -60,9 +60,11 @@ function PubSubProducer() { const topicList = await pubSubClient.getTopics(); if (topicList) { - topicList[0].forEach(topic => { - topics.push(getName(topic.name)); - }); + if (topicList) { + topicList[0].forEach(topic => { + topics.push(getName(topic.name)); + }); + } } const subscriptionList = await pubSubClient.getSubscriptions(); From 8d5c38b743a91c2ad3739c25c47f93ece5480f08 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 1 May 2020 17:43:13 +0300 Subject: [PATCH 151/196] Queue refactoring --- .../processing/AbstractConsumerService.java | 1 - .../BatchTbRuleEngineSubmitStrategy.java | 5 +- .../BurstTbRuleEngineSubmitStrategy.java | 10 +- ...lByEntityIdTbRuleEngineSubmitStrategy.java | 8 - ...lByTenantIdTbRuleEngineSubmitStrategy.java | 1 - .../SequentialTbRuleEngineSubmitStrategy.java | 6 +- ...TbRuleEngineProcessingStrategyFactory.java | 8 +- .../TbServiceBusConsumerTemplate.java | 116 +++++--------- ...stractParallelTbQueueConsumerTemplate.java | 53 +++++++ .../AbstractTbQueueConsumerTemplate.java | 20 ++- .../queue/kafka/TbKafkaConsumerTemplate.java | 3 +- .../pubsub/TbPubSubConsumerTemplate.java | 102 ++++--------- .../rabbitmq/TbRabbitMqConsumerTemplate.java | 107 ++++--------- .../queue/sqs/TbAwsSqsConsumerTemplate.java | 142 ++++++------------ 14 files changed, 232 insertions(+), 350 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index c2705fcbdc..4007c9e17d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -23,7 +23,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java index d0b1f7f99a..b9741d2433 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -23,7 +23,6 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -77,8 +76,8 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS } } int submitSize = pendingPack.size(); - if (log.isInfoEnabled() && submitSize > 0) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + if (log.isDebugEnabled() && submitSize > 0) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, submitSize); } pendingPack.forEach(msgConsumer); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java index ffd1dd49d1..3420933d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -19,14 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; @Slf4j public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { @@ -37,8 +31,8 @@ public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS @Override public void submitAttempt(BiConsumer> msgConsumer) { - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); } orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java index ae5993cb1c..473810b86c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -15,26 +15,18 @@ */ package org.thingsboard.server.service.queue.processing; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; -import java.util.stream.Collectors; @Slf4j public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java index b258c6db1b..37e9419edd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -30,6 +30,5 @@ public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialBy @Override protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java index ef45b983fc..125a1d8ef8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @@ -63,8 +61,8 @@ public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSu if (idx < listSize) { IdMsgPair pair = orderedMsgList.get(idx); expectedMsgId = pair.uuid; - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg); } msgConsumer.accept(pair.uuid, pair.msg); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index bbf283e962..80b0523a81 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -82,10 +82,10 @@ public class TbRuleEngineProcessingStrategyFactory { retryCount++; double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { - log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); + log.debug("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); + log.debug("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); return new TbRuleEngineProcessingDecision(true, null); } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); @@ -98,7 +98,7 @@ public class TbRuleEngineProcessingStrategyFactory { if (retrySuccessful) { result.getSuccessMap().forEach(toReprocess::put); } - log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); + log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); } @@ -126,7 +126,7 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { if (!result.isSuccess()) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); } if (log.isTraceEnabled()) { result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java index cca599d59a..4db5ade728 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -31,9 +31,9 @@ import org.apache.qpid.proton.amqp.transport.SenderSettleMode; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.time.Duration; @@ -50,102 +50,72 @@ import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j -public class TbServiceBusConsumerTemplate implements TbQueueConsumer { +public class TbServiceBusConsumerTemplate extends AbstractTbQueueConsumerTemplate { private final TbQueueAdmin admin; - private final String topic; private final TbQueueMsgDecoder decoder; private final TbServiceBusSettings serviceBusSettings; private final Gson gson = new Gson(); private Set receivers; - private volatile Set partitions; - private volatile boolean subscribed; - private volatile boolean stopped = false; private Map> pendingMessages = new ConcurrentHashMap<>(); private volatile int messagesPerQueue; public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; this.serviceBusSettings = serviceBusSettings; } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", getTopic()); + } else { + log.error("Failed to receive messages", e); + } + return Collections.emptyList(); + } } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); } @Override - public void unsubscribe() { - stopped = true; + protected void doUnsubscribe() { receivers.forEach(CoreMessageReceiver::closeAsync); } - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - createReceivers(); - messagesPerQueue = receivers.size() / partitions.size(); - subscribed = true; - } - - List>> messageFutures = - receivers.stream() - .map(receiver -> receiver - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) - .whenComplete((messages, err) -> { - if (!CollectionUtils.isEmpty(messages)) { - pendingMessages.put(receiver, messages); - } else if (err != null) { - log.error("Failed to receive messages.", err); - } - })) - .collect(Collectors.toList()); - try { - return fromList(messageFutures) - .get() - .stream() - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) - .map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to parse message.", e); - throw new RuntimeException("Failed to parse message.", e); - } - }).collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - private void createReceivers() { List> receiverFutures = partitions.stream() .map(TopicPartitionInfo::getFullTopicName) @@ -167,7 +137,7 @@ public class TbServiceBusConsumerTemplate implements TbQue receivers = new HashSet<>(fromList(receiverFutures).get()); } catch (InterruptedException | ExecutionException e) { if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); + log.info("[{}] Service Bus consumer is stopped.", getTopic()); } else { log.error("Failed to create receivers", e); } @@ -196,13 +166,7 @@ public class TbServiceBusConsumerTemplate implements TbQue } @Override - public void commit() { - pendingMessages.forEach((receiver, msgs) -> - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); - pendingMessages.clear(); - } - - private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + protected T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java new file mode 100644 index 0000000000..bb83a79250 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class AbstractParallelTbQueueConsumerTemplate extends AbstractTbQueueConsumerTemplate { + + protected ListeningExecutorService consumerExecutor; + + public AbstractParallelTbQueueConsumerTemplate(String topic) { + super(topic); + } + + protected void initNewExecutor(int threadPoolSize) { + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + try { + consumerExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + log.trace("Interrupted while waiting for consumer executor to stop"); + } + } + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadPoolSize)); + } + + protected void shutdownExecutor() { + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 084d10fca9..c8cc545601 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -28,11 +28,13 @@ import java.util.List; import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; @Slf4j public abstract class AbstractTbQueueConsumerTemplate implements TbQueueConsumer { private volatile boolean subscribed; + protected volatile boolean stopped = false; protected volatile Set partitions; protected final Lock consumerLock = new ReentrantLock(); @@ -74,10 +76,12 @@ public abstract class AbstractTbQueueConsumerTemplate i log.debug("Failed to await subscription", e); } } else { + long pollStartTs = System.currentTimeMillis(); consumerLock.lock(); try { if (!subscribed) { - doSubscribe(); + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + doSubscribe(topicNames); subscribed = true; } @@ -95,6 +99,17 @@ public abstract class AbstractTbQueueConsumerTemplate i } }); return result; + } else { + long pollDuration = System.currentTimeMillis() - pollStartTs; + if (pollDuration < durationInMillis) { + try { + Thread.sleep(durationInMillis - pollDuration); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + } } } finally { consumerLock.unlock(); @@ -115,6 +130,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { + stopped = true; consumerLock.lock(); try { doUnsubscribe(); @@ -127,7 +143,7 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected T decode(R record) throws IOException; - abstract protected void doSubscribe(); + abstract protected void doSubscribe(List topicNames); abstract protected void doCommit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index fea31854df..75635de7a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -69,8 +69,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue } @Override - protected void doSubscribe() { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + protected void doSubscribe( List topicNames) { topicNames.forEach(admin::createTopicIfNotExists); consumer.subscribe(topicNames); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 5dc795739a..7302d19ff7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.pubsub; +import com.amazonaws.services.sqs.model.Message; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; @@ -35,11 +36,14 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -47,10 +51,11 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j -public class TbPubSubConsumerTemplate implements TbQueueConsumer { +public class TbPubSubConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { private final Gson gson = new Gson(); private final TbQueueAdmin admin; @@ -58,23 +63,18 @@ public class TbPubSubConsumerTemplate implements TbQueueCo private final TbQueueMsgDecoder decoder; private final TbPubSubSettings pubSubSettings; - private volatile boolean subscribed; - private volatile Set partitions; private volatile Set subscriptionNames; private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); - private ExecutorService consumerExecutor; private final SubscriberStub subscriber; - private volatile boolean stopped; - private volatile int messagesPerTopic; public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.pubSubSettings = pubSubSettings; this.topic = topic; this.decoder = decoder; - try { SubscriberStubSettings subscriberStubSettings = SubscriberStubSettings.newBuilder() @@ -84,89 +84,50 @@ public class TbPubSubConsumerTemplate implements TbQueueCo .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) .build()) .build(); - this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); } catch (IOException e) { log.error("Failed to create subscriber.", e); throw new RuntimeException("Failed to create subscriber.", e); } - stopped = false; } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + try { + List messages = receiveMessages(); + if (!messages.isEmpty()) { + return messages.stream().map(ReceivedMessage::getMessage).collect(Collectors.toList()); + } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); + } + } + return Collections.emptyList(); } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + subscriptionNames = new LinkedHashSet<>(topicNames); + subscriptionNames.forEach(admin::createTopicIfNotExists); + initNewExecutor(subscriptionNames.size() + 1); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); + acknowledgeRequests.clear(); } @Override - public void unsubscribe() { - stopped = true; - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - + protected void doUnsubscribe() { if (subscriber != null) { subscriber.close(); } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); - subscriptionNames.forEach(admin::createTopicIfNotExists); - consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); - subscribed = true; - } - List messages; - try { - messages = receiveMessages(); - if (!messages.isEmpty()) { - List result = new ArrayList<>(); - messages.forEach(msg -> { - try { - result.add(decode(msg.getMessage())); - } catch (InvalidProtocolBufferException e) { - log.error("Failed decode record: [{}]", msg); - } - }); - return result; - } - } catch (ExecutionException | InterruptedException e) { - if (stopped) { - log.info("[{}] Pub/Sub consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); - acknowledgeRequests.clear(); + shutdownExecutor(); } private List receiveMessages() throws ExecutionException, InterruptedException { @@ -211,6 +172,7 @@ public class TbPubSubConsumerTemplate implements TbQueueCo return transform.get(); } + @Override public T decode(PubsubMessage message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); return decoder.decode(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java index 25d7719163..45dc9d6a05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -23,9 +23,9 @@ import com.rabbitmq.client.GetResponse; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; @@ -37,33 +37,26 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @Slf4j -public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { +public class TbRabbitMqConsumerTemplate extends AbstractTbQueueConsumerTemplate { private final Gson gson = new Gson(); private final TbQueueAdmin admin; - private final String topic; private final TbQueueMsgDecoder decoder; - private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; - private volatile Set partitions; - private volatile boolean subscribed; private volatile Set queues; - private volatile boolean stopped; public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; - this.rabbitMqSettings = rabbitMqSettings; try { connection = rabbitMqSettings.getConnectionFactory().newConnection(); } catch (IOException | TimeoutException e) { log.error("Failed to create connection.", e); throw new RuntimeException("Failed to create connection.", e); } - try { channel = connection.createChannel(); } catch (IOException e) { @@ -74,25 +67,42 @@ public class TbRabbitMqConsumerTemplate implements TbQueue } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } else { + return Collections.emptyList(); + } } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + queues.forEach(admin::createTopicIfNotExists); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } } @Override - public void unsubscribe() { - stopped = true; + protected void doUnsubscribe() { if (channel != null) { try { channel.close(); @@ -109,63 +119,6 @@ public class TbRabbitMqConsumerTemplate implements TbQueue } } - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - queues = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.toSet()); - - queues.forEach(admin::createTopicIfNotExists); - subscribed = true; - } - - List result = queues.stream() - .map(queue -> { - try { - return channel.basicGet(queue, false); - } catch (IOException e) { - log.error("Failed to get messages from queue: [{}]", queue); - throw new RuntimeException("Failed to get messages from queue.", e); - } - }).filter(Objects::nonNull).map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to decode message: [{}].", message); - throw new RuntimeException("Failed to decode message.", e); - } - }).collect(Collectors.toList()); - if (result.size() > 0) { - return result; - } - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to wait.", e); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - channel.basicAck(0, true); - } catch (IOException e) { - log.error("Failed to ack messages.", e); - } - } - public T decode(GetResponse message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); return decoder.decode(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 3e71388844..317dd93902 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -25,21 +25,17 @@ import com.amazonaws.services.sqs.model.Message; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,34 +43,28 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j -public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { +public class TbAwsSqsConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { private static final int MAX_NUM_MSGS = 10; private final Gson gson = new Gson(); private final TbQueueAdmin admin; private final AmazonSQS sqsClient; - private final String topic; private final TbQueueMsgDecoder decoder; private final TbAwsSqsSettings sqsSettings; private final List pendingMessages = new CopyOnWriteArrayList<>(); private volatile Set queueUrls; - private volatile Set partitions; - private ListeningExecutorService consumerExecutor; - private volatile boolean subscribed; - private volatile boolean stopped = false; public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; this.sqsSettings = sqsSettings; AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); @@ -87,81 +77,64 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo } @Override - public String getTopic() { - return topic; + protected void doSubscribe(List topicNames) { + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + initNewExecutor(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1); } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected List doPoll(long durationInMillis) { + if (!pendingMessages.isEmpty()) { + log.warn("Present {} non committed messages.", pendingMessages.size()); + return Collections.emptyList(); + } + int duration = (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis); + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, duration)) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", getTopic()); + } else { + log.error("Failed to pool messages.", e); + } + return Collections.emptyList(); + } } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + public T decode(Message message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); + return decoder.decode(msg); } @Override - public void unsubscribe() { - stopped = true; - - if (sqsClient != null) { - sqsClient.shutdown(); - } - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } + protected void doCommit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + pendingMessages.clear(); } @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); - consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); - subscribed = true; - } - - if (!pendingMessages.isEmpty()) { - log.warn("Present {} non committed messages.", pendingMessages.size()); - return Collections.emptyList(); - } - - List>> futureList = queueUrls - .stream() - .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) - .collect(Collectors.toList()); - ListenableFuture>> futureResult = Futures.allAsList(futureList); - try { - return futureResult.get().stream() - .flatMap(List::stream) - .map(msg -> { - try { - return decode(msg); - } catch (IOException e) { - log.error("Failed to decode message: [{}]", msg); - return null; - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Aws SQS consumer is stopped.", topic); - } else { - log.error("Failed to pool messages.", e); - } - } + protected void doUnsubscribe() { + stopped = true; + if (sqsClient != null) { + sqsClient.shutdown(); } - return Collections.emptyList(); + shutdownExecutor(); } private ListenableFuture> poll(String url, int waitTimeSeconds) { @@ -194,25 +167,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo }, consumerExecutor); } - @Override - public void commit() { - pendingMessages.forEach(msg -> - consumerExecutor.submit(() -> { - List entries = msg.getMessages() - .stream() - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) - .collect(Collectors.toList()); - sqsClient.deleteMessageBatch(msg.getUrl(), entries); - })); - - pendingMessages.clear(); - } - - public T decode(Message message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - @Data private static class AwsSqsMsgWrapper { private final String url; From 06c3caf082ee48eed660927ff453d530e965bfb6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sat, 2 May 2020 13:34:11 +0300 Subject: [PATCH 152/196] refactored --- .../server/queue/pubsub/TbPubSubAdmin.java | 32 ++++++++++++----- .../pubsub/TbPubSubConsumerTemplate.java | 5 +++ .../pubsub/TbPubSubProducerTemplate.java | 4 +-- msa/js-executor/queue/pubSubTemplate.js | 35 +++++++++++-------- pom.xml | 2 +- 5 files changed, 52 insertions(+), 26 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index 1241370d05..d0a514ffd3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.pubsub; +import com.google.api.gax.rpc.AlreadyExistsException; import com.google.cloud.pubsub.v1.SubscriptionAdminClient; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminClient; @@ -24,9 +25,9 @@ import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; @@ -103,7 +104,10 @@ public class TbPubSubAdmin implements TbQueueAdmin { @Override public void createTopicIfNotExists(String partition) { - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + TopicName topicName = TopicName.newBuilder() + .setTopic(partition) + .setProject(pubSubSettings.getProjectId()) + .build(); if (topicSet.contains(topicName.toString())) { createSubscriptionIfNotExists(partition, topicName); @@ -121,13 +125,18 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); + try { + topicAdminClient.createTopic(topicName); + log.info("Created new topic: [{}]", topicName.toString()); + } catch (AlreadyExistsException e) { + log.info("[{}] Topic already exist.", topicName.toString()); + } finally { + topicSet.add(topicName.toString()); + } createSubscriptionIfNotExists(partition, topicName); } - private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + private void createSubscriptionIfNotExists(String partition, TopicName topicName) { ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); @@ -153,9 +162,14 @@ public class TbPubSubAdmin implements TbQueueAdmin { setAckDeadline(subscriptionBuilder); setMessageRetention(subscriptionBuilder); - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); + try { + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } catch (AlreadyExistsException e) { + log.info("[{}] Subscription already exist.", subscriptionName.toString()); + } finally { + subscriptionSet.add(subscriptionName.toString()); + } } private void setAckDeadline(Subscription.Builder builder) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 5dc795739a..495c895a91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -134,6 +134,11 @@ public class TbPubSubConsumerTemplate implements TbQueueCo if (!subscribed) { subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); subscriptionNames.forEach(admin::createTopicIfNotExists); + + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + } + consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); subscribed = true; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 2cd2e1054e..7a073616fd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -124,8 +124,8 @@ public class TbPubSubProducerTemplate implements TbQueuePr publisherMap.put(topic, publisher); return publisher; } catch (IOException e) { - log.error("Failed to create topic [{}].", topic, e); - throw new RuntimeException("Failed to create topic.", e); + log.error("Failed to create Publisher for the topic [{}].", topic, e); + throw new RuntimeException("Failed to create Publisher for the topic.", e); } } diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index cc5284022d..8cb264de64 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -60,11 +60,9 @@ function PubSubProducer() { const topicList = await pubSubClient.getTopics(); if (topicList) { - if (topicList) { - topicList[0].forEach(topic => { - topics.push(getName(topic.name)); - }); - } + topicList[0].forEach(topic => { + topics.push(getName(topic.name)); + }); } const subscriptionList = await pubSubClient.getSubscriptions(); @@ -100,23 +98,32 @@ function PubSubProducer() { async function createTopic(topic) { if (!topics.includes(topic)) { - await pubSubClient.createTopic(topic); + try { + await pubSubClient.createTopic(topic); + logger.info('Created new Pub/Sub topic: %s', topic); + } catch (e) { + logger.info('Pub/Sub topic already exists'); + } topics.push(topic); - logger.info('Created new Pub/Sub topic: %s', topic); } await createSubscription(topic) } async function createSubscription(topic) { if (!subscriptions.includes(topic)) { - await pubSubClient.createSubscription(topic, topic, { - topic: topic, - subscription: topic, - ackDeadlineSeconds: queueProps['ackDeadlineInSec'], - messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} - }); + try { + await pubSubClient.createSubscription(topic, topic, { + topic: topic, + subscription: topic, + ackDeadlineSeconds: queueProps['ackDeadlineInSec'], + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} + }); + logger.info('Created new Pub/Sub subscription: %s', topic); + } catch (e) { + logger.info('Pub/Sub subscription already exists.'); + } + subscriptions.push(topic); - logger.info('Created new Pub/Sub subscription: %s', topic); } } diff --git a/pom.xml b/pom.xml index 494a43cb56..a1dbc2d8f5 100755 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ 1.25 1.3.10 1.11.747 - 1.84.0 + 1.105.0 3.2.0 1.5.0 1.4.3 From e251bc575036753744daa321bd2ca6017665b0bd Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Sat, 2 May 2020 18:12:43 +0300 Subject: [PATCH 153/196] Improvement to deserialization and remove debug on dashboards --- .../data/json/demo/rule_chains/root_rule_chain.json | 4 ++-- .../json/demo/rule_chains/thermostat_alarms.json | 4 ++-- .../org/thingsboard/server/common/msg/TbMsg.java | 12 +++++++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 32cecf6e4e..9805c6f996 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -17,7 +17,7 @@ }, "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", "name": "Is Thermostat?", - "debugMode": true, + "debugMode": false, "configuration": { "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";" } @@ -113,7 +113,7 @@ }, "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode", "name": "Relate to Asset", - "debugMode": true, + "debugMode": false, "configuration": { "direction": "FROM", "relationType": "ToAlarmPropagationAsset", diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json index 8fd10c88f1..d67052cbc5 100644 --- a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -81,7 +81,7 @@ }, "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", "name": "Check Alarms", - "debugMode": true, + "debugMode": false, "configuration": { "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" } @@ -93,7 +93,7 @@ }, "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", "name": "Fetch Configuration", - "debugMode": true, + "debugMode": false, "configuration": { "clientAttributeNames": [], "sharedAttributeNames": [], diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 3edf4e5061..9da7407552 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import java.io.IOException; import java.io.Serializable; import java.util.UUID; @@ -47,7 +48,7 @@ public final class TbMsg implements Serializable { private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; //This field is not serialized because we use queues and there is no need to do it - private final TbMsgCallback callback; + transient private final TbMsgCallback callback; public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); @@ -156,4 +157,13 @@ public final class TbMsg implements Serializable { public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); } + + public TbMsgCallback getCallback() { + //May be null in case of deserialization; + if (callback != null) { + return callback; + } else { + return TbMsgCallback.EMPTY; + } + } } From d2919ba30e02df8af283d0397f133281ae50a088 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Sun, 3 May 2020 02:17:17 +0300 Subject: [PATCH 154/196] Improvement to Clear Alarm Node --- .../org/thingsboard/rule/engine/action/TbClearAlarmNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java index 63fb59bed5..b0413c6dd7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java @@ -81,8 +81,8 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode Date: Mon, 4 May 2020 00:13:14 +0300 Subject: [PATCH 155/196] Better logging of Rule Engine errors --- .../RuleChainActorMessageProcessor.java | 29 ++++++++++++++----- .../actors/ruleChain/RuleNodeActor.java | 12 +++++--- .../RuleNodeActorMessageProcessor.java | 24 +++++++-------- .../actors/shared/ComponentMsgProcessor.java | 13 +++++++-- .../common/msg/queue/RuleNodeException.java | 12 ++++++-- 5 files changed, 61 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 4e3d69c2b7..61c5c0da28 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -163,7 +163,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes, String failureMessage) { try { - checkActive(); + checkActive(msg); EntityId entityId = msg.getOriginator(); TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); List relations = nodeRoutes.get(originatorNodeId).stream() @@ -272,6 +278,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor { + private final String ruleChainName; private final RuleChainId ruleChainId; - private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, String ruleChainName, RuleNodeId ruleNodeId) { super(systemContext, tenantId, ruleNodeId); + this.ruleChainName = ruleChainName; this.ruleChainId = ruleChainId; - setProcessor(new RuleNodeActorMessageProcessor(tenantId, ruleChainId, ruleNodeId, systemContext, + setProcessor(new RuleNodeActorMessageProcessor(tenantId, this.ruleChainName, ruleNodeId, systemContext, context().parent(), context().self())); } @@ -96,19 +98,21 @@ public class RuleNodeActor extends ComponentActor { - private final ActorRef parent; + private final String ruleChainName; private final ActorRef self; - private final RuleChainService service; private RuleNode ruleNode; private TbNode tbNode; private DefaultTbContext defaultCtx; - RuleNodeActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, ActorSystemContext systemContext + RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { super(systemContext, tenantId, ruleNodeId); - this.parent = parent; + this.ruleChainName = ruleChainName; this.self = self; - this.service = systemContext.getRuleChainService(); this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); this.defaultCtx = new DefaultTbContext(systemContext, new RuleNodeCtx(tenantId, parent, self, ruleNode)); } @@ -63,8 +59,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -74,11 +77,17 @@ public abstract class ComponentMsgProcessor extends Abstract schedulePeriodicMsgWithDelay(context, new StatsPersistTick(), statsPersistFrequency, statsPersistFrequency); } - protected void checkActive() { + protected void checkActive(TbMsg tbMsg) throws RuleNodeException { if (state != ComponentLifecycleState.ACTIVE) { log.debug("Component is not active. Current state [{}] for processor [{}][{}] tenant [{}]", state, entityId.getEntityType(), entityId, tenantId); - throw new IllegalStateException("Rule chain is not active! " + entityId + " - " + tenantId); + RuleNodeException ruleNodeException = getInactiveException(); + if (tbMsg != null) { + tbMsg.getCallback().onFailure(ruleNodeException); + } + throw ruleNodeException; } } + abstract protected RuleNodeException getInactiveException(); + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java index 3f437dd1d7..288e1b6002 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -36,9 +36,15 @@ public class RuleNodeException extends RuleEngineException { public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { super(message); this.ruleChainName = ruleChainName; - this.ruleNodeName = ruleNode.getName(); - this.ruleChainId = ruleNode.getRuleChainId(); - this.ruleNodeId = ruleNode.getId(); + if (ruleNode != null) { + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } else { + ruleNodeName = "Unknown"; + ruleChainId = new RuleChainId(RuleChainId.NULL_UUID); + ruleNodeId = new RuleNodeId(RuleNodeId.NULL_UUID); + } } public String toJsonString() { From 7fc46010b790f0453a14a70ffba77383351e8734 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 29 Apr 2020 16:37:03 +0300 Subject: [PATCH 156/196] Added base impl for OAuth-2 --- application/pom.xml | 12 ++ .../ThingsboardOAuth2Configuration.java | 81 ++++++++++++ .../ThingsboardSecurityConfiguration.java | 22 ++- .../Oauth2AuthenticationSuccessHandler.java | 125 ++++++++++++++++++ ...RestAwareAuthenticationSuccessHandler.java | 2 +- application/src/main/resources/logback.xml | 1 + .../src/main/resources/thingsboard.yml | 38 ++++++ pom.xml | 15 +++ ui/src/app/login/login.tpl.html | 1 + 9 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java diff --git a/application/pom.xml b/application/pom.xml index 21a319491d..55009709d1 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -132,6 +132,18 @@ org.springframework.boot spring-boot-starter-websocket + + org.springframework.cloud + spring-cloud-starter-oauth2 + + + org.springframework.security + spring-security-oauth2-client + + + org.springframework.security + spring-security-oauth2-jose + io.jsonwebtoken jjwt diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java new file mode 100644 index 0000000000..45048fddf3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.oauth2.client.registration.ClientRegistration; +import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; +import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; +import org.springframework.security.oauth2.core.AuthorizationGrantType; +import org.springframework.security.oauth2.core.ClientAuthenticationMethod; + +import java.util.Collections; + +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +@Configuration +public class ThingsboardOAuth2Configuration { + + @Value("${security.oauth2.registrationId}") + private String registrationId; + @Value("${security.oauth2.userNameAttributeName}") + private String userNameAttributeName; + + @Value("${security.oauth2.client.clientId}") + private String clientId; + @Value("${security.oauth2.client.clientName}") + private String clientName; + @Value("${security.oauth2.client.clientSecret}") + private String clientSecret; + @Value("${security.oauth2.client.accessTokenUri}") + private String accessTokenUri; + @Value("${security.oauth2.client.authorizationUri}") + private String authorizationUri; + @Value("${security.oauth2.client.redirectUriTemplate}") + private String redirectUriTemplate; + @Value("${security.oauth2.client.scope}") + private String scope; + @Value("${security.oauth2.client.jwkSetUri}") + private String jwkSetUri; + @Value("${security.oauth2.client.authorizationGrantType}") + private String authorizationGrantType; + @Value("${security.oauth2.client.clientAuthenticationMethod}") + private String clientAuthenticationMethod; + + @Value("${security.oauth2.resource.userInfoUri}") + private String userInfoUri; + + @Bean + public ClientRegistrationRepository clientRegistrationRepository() { + ClientRegistration registration = ClientRegistration.withRegistrationId(registrationId) + .clientId(clientId) + .authorizationUri(authorizationUri) + .clientSecret(clientSecret) + .tokenUri(accessTokenUri) + .redirectUriTemplate(redirectUriTemplate) + .scope(scope.split(",")) + .clientName(clientName) + .authorizationGrantType(new AuthorizationGrantType(authorizationGrantType)) + .userInfoUri(userInfoUri) + .userNameAttributeName(userNameAttributeName) + .jwkSetUri(jwkSetUri) + .clientAuthenticationMethod(new ClientAuthenticationMethod(clientAuthenticationMethod)) + .build(); + return new InMemoryClientRegistrationRepository(Collections.singletonList(registration)); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 53876f4040..36490b5166 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -18,6 +18,8 @@ package org.thingsboard.server.config; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.security.SecurityProperties; import org.springframework.context.annotation.Bean; @@ -73,12 +75,25 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt public static final String WS_TOKEN_BASED_AUTH_ENTRY_POINT = "/api/ws/**"; @Autowired private ThingsboardErrorResponseHandler restAccessDeniedHandler; - @Autowired private AuthenticationSuccessHandler successHandler; + + @Autowired(required = false) + @Qualifier("oauth2AuthenticationSuccessHandler") + private AuthenticationSuccessHandler oauth2AuthenticationSuccessHandler; + + @Autowired + @Qualifier("defaultAuthenticationSuccessHandler") + private AuthenticationSuccessHandler successHandler; + @Autowired private AuthenticationFailureHandler failureHandler; @Autowired private RestAuthenticationProvider restAuthenticationProvider; @Autowired private JwtAuthenticationProvider jwtAuthenticationProvider; @Autowired private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider; + @Value("${security.oauth2.enabled}") + private boolean oauth2Enabled; + @Value("${security.oauth2.client.loginProcessingUrl}") + private String loginProcessingUrl; + @Autowired @Qualifier("jwtHeaderTokenExtractor") private TokenExtractor jwtHeaderTokenExtractor; @@ -189,6 +204,11 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); + if (oauth2Enabled) { + http.oauth2Login() + .loginProcessingUrl(loginProcessingUrl) + .successHandler(oauth2AuthenticationSuccessHandler); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java new file mode 100644 index 0000000000..e99e5896e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.authentication.InsufficientAuthenticationException; +import org.springframework.security.authentication.LockedException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; +import org.thingsboard.server.service.security.model.token.JwtToken; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; +import org.thingsboard.server.service.security.system.SystemSecurityService; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@Component(value="oauth2AuthenticationSuccessHandler") +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { + + private final ObjectMapper mapper; + private final JwtTokenFactory tokenFactory; + private final RefreshTokenRepository refreshTokenRepository; + private final SystemSecurityService systemSecurityService; + private final UserService userService; + + @Autowired + public Oauth2AuthenticationSuccessHandler(final ObjectMapper mapper, + final JwtTokenFactory tokenFactory, + final RefreshTokenRepository refreshTokenRepository, + final UserService userService, + final SystemSecurityService systemSecurityService) { + this.mapper = mapper; + this.tokenFactory = tokenFactory; + this.refreshTokenRepository = refreshTokenRepository; + this.userService = userService; + this.systemSecurityService = systemSecurityService; + } + + @Override + public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { + Object object = authentication.getPrincipal(); + + System.out.println(object); + + // active user check + + UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, "tenant@thingsboard.org"); + SecurityUser securityUser = (SecurityUser) authenticateByUsernameAndPassword(principal,"tenant@thingsboard.org", "tenant").getPrincipal(); + + JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); + JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); + + Map tokenMap = new HashMap(); + tokenMap.put("token", accessToken.getToken()); + tokenMap.put("refreshToken", refreshToken.getToken()); + +// response.setStatus(HttpStatus.OK.value()); +// response.setContentType(MediaType.APPLICATION_JSON_VALUE); +// mapper.writeValue(response.getWriter(), tokenMap); + + request.setAttribute("token", accessToken.getToken()); + response.addHeader("token", accessToken.getToken()); + + getRedirectStrategy().sendRedirect(request, response, "http://localhost:4200/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); + } + + private Authentication authenticateByUsernameAndPassword(UserPrincipal userPrincipal, String username, String password) { + User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, username); + if (user == null) { + throw new UsernameNotFoundException("User not found: " + username); + } + + try { + + UserCredentials userCredentials = userService.findUserCredentialsByUserId(TenantId.SYS_TENANT_ID, user.getId()); + if (userCredentials == null) { + throw new UsernameNotFoundException("User credentials not found"); + } + + try { + systemSecurityService.validateUserCredentials(user.getTenantId(), userCredentials, username, password); + } catch (LockedException e) { + throw e; + } + + if (user.getAuthority() == null) + throw new InsufficientAuthenticationException("User has no authority assigned"); + + SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), userPrincipal); + return new UsernamePasswordAuthenticationToken(securityUser, null, securityUser.getAuthorities()); + } catch (Exception e) { + throw e; + } + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index aa55818084..d26b02174e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -@Component +@Component(value="defaultAuthenticationSuccessHandler") public class RestAwareAuthenticationSuccessHandler implements AuthenticationSuccessHandler { private final ObjectMapper mapper; private final JwtTokenFactory tokenFactory; diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e25fb72ccb..9b14ff06f5 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -26,6 +26,7 @@ + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bf82c586c3..97e22fa2c6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -97,6 +97,44 @@ security: allowClaimingByDefault: "${SECURITY_CLAIM_ALLOW_CLAIMING_BY_DEFAULT:true}" # Time allowed to claim the device in milliseconds duration: "${SECURITY_CLAIM_DURATION:60000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value + basic: + enabled: false + # oauth2: + # enabled: true + # registrationId: A + # userNameAttributeName: email + # client: + # clientName: Thingsboard Dev Test Q + # clientId: 5f5c0998-1d9b-4679-9610-6108fb91af2a + # clientSecret: h_kXVb7Ee1LgDDinix_nkAh_owWX7YCO783NNteF9AIOqlTWu2L03YoFjv5KL8yRVyx4uYAE-r_N3tFbupE8Kw + # accessTokenUri: https://federation-q.auth.schwarz/nidp/oauth/nam/token + # authorizationUri: https://federation-q.auth.schwarz/nidp/oauth/nam/authz + # scope: openid,profile,email,siam + # redirectUriTemplate: http://localhost:8080/login/oauth2/code/ + # loginProcessingUrl: /login/oauth2/code/ + # jwkSetUri: https://federation-q.auth.schwarz/nidp/oauth/nam/keys + # authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials + # clientAuthenticationMethod: post # basic, post + # resource: + # userInfoUri: https://federation-q.auth.schwarz/nidp/oauth/nam/userinfo + oauth2: + enabled: true + registrationId: A + userNameAttributeName: email + client: + clientName: Test app + clientId: dVH9reqyqiXIG7M2wmamb0ySue8zaM4g + clientSecret: EYAfAGxwkwoeYnb2o2cDgaWZB5k97OStpZQPPvcMMD-SVH2BuughTGeBazXtF5I6 + accessTokenUri: https://dev-r9m8ht0k.auth0.com/oauth/token + authorizationUri: https://dev-r9m8ht0k.auth0.com/authorize + scope: openid,profile,email + redirectUriTemplate: http://localhost:8080/login/oauth2/code/ + loginProcessingUrl: /login/oauth2/code/ + jwkSetUri: https://dev-r9m8ht0k.auth0.com/.well-known/jwks.json + authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials + clientAuthenticationMethod: post # basic, post + resource: + userInfoUri: https://dev-r9m8ht0k.auth0.com/userinfo # Dashboard parameters dashboard: diff --git a/pom.xml b/pom.xml index 494a43cb56..f263307538 100755 --- a/pom.xml +++ b/pom.xml @@ -458,6 +458,21 @@ spring-boot-starter-security ${spring-boot.version} + + org.springframework.cloud + spring-cloud-starter-oauth2 + ${spring-boot.version} + + + org.springframework.security + spring-security-oauth2-client + ${spring.version} + + + org.springframework.security + spring-security-oauth2-jose + ${spring.version} + org.springframework.boot spring-boot-starter-web diff --git a/ui/src/app/login/login.tpl.html b/ui/src/app/login/login.tpl.html index 6e9d7d8977..409f5cd4ad 100644 --- a/ui/src/app/login/login.tpl.html +++ b/ui/src/app/login/login.tpl.html @@ -47,6 +47,7 @@
{{ 'login.login' | translate }} + OAUTH2 LOGIN
From a563fdab3f1a1e37338190bbf933d55268551fec Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 1 May 2020 03:28:44 +0300 Subject: [PATCH 157/196] Added basic and custom OAuth2 user mappers --- .../ThingsboardOAuth2Configuration.java | 81 ------------ .../ThingsboardSecurityConfiguration.java | 14 +- .../server/controller/AuthController.java | 15 +++ .../Oauth2AuthenticationSuccessHandler.java | 125 ------------------ .../auth/oauth2/BaseOAuth2ClientMapper.java | 112 ++++++++++++++++ .../auth/oauth2/BasicOAuth2ClientMapper.java | 79 +++++++++++ .../auth/oauth2/CustomOAuth2ClientMapper.java | 47 +++++++ .../auth/oauth2/OAuth2ClientMapper.java | 24 ++++ .../oauth2/OAuth2ClientMapperProvider.java | 45 +++++++ .../Oauth2AuthenticationSuccessHandler.java | 70 ++++++++++ ...RestAwareAuthenticationSuccessHandler.java | 2 +- .../src/main/resources/thingsboard.yml | 77 +++++++---- .../server/dao/oauth2/OAuth2Service.java | 25 ++++ .../server/dao/oauth2/OAuth2User.java | 27 ++++ .../common/data/id/OAuth2IntegrationId.java | 35 +++++ .../common/data/oauth2/OAuth2ClientInfo.java | 45 +++++++ dao/pom.xml | 4 + .../server/dao/oauth2/OAuth2Client.java | 41 ++++++ .../dao/oauth2/OAuth2ClientMapperConfig.java | 44 ++++++ .../dao/oauth2/OAuth2Configuration.java | 80 +++++++++++ .../server/dao/oauth2/OAuth2ServiceImpl.java | 49 +++++++ pom.xml | 3 +- ui/src/app/api/login.service.js | 15 ++- ui/src/app/app.run.js | 35 ++++- ui/src/app/locale/locale.constant-cs_CZ.json | 6 +- ui/src/app/locale/locale.constant-en_US.json | 6 +- ui/src/app/locale/locale.constant-ru_RU.json | 4 +- ui/src/app/locale/locale.constant-uk_UA.json | 6 +- ui/src/app/login/login.scss | 36 +++++ ui/src/app/login/login.tpl.html | 23 +++- 30 files changed, 916 insertions(+), 259 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java deleted file mode 100644 index 45048fddf3..0000000000 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.config; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.oauth2.client.registration.ClientRegistration; -import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; -import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; -import org.springframework.security.oauth2.core.AuthorizationGrantType; -import org.springframework.security.oauth2.core.ClientAuthenticationMethod; - -import java.util.Collections; - -@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") -@Configuration -public class ThingsboardOAuth2Configuration { - - @Value("${security.oauth2.registrationId}") - private String registrationId; - @Value("${security.oauth2.userNameAttributeName}") - private String userNameAttributeName; - - @Value("${security.oauth2.client.clientId}") - private String clientId; - @Value("${security.oauth2.client.clientName}") - private String clientName; - @Value("${security.oauth2.client.clientSecret}") - private String clientSecret; - @Value("${security.oauth2.client.accessTokenUri}") - private String accessTokenUri; - @Value("${security.oauth2.client.authorizationUri}") - private String authorizationUri; - @Value("${security.oauth2.client.redirectUriTemplate}") - private String redirectUriTemplate; - @Value("${security.oauth2.client.scope}") - private String scope; - @Value("${security.oauth2.client.jwkSetUri}") - private String jwkSetUri; - @Value("${security.oauth2.client.authorizationGrantType}") - private String authorizationGrantType; - @Value("${security.oauth2.client.clientAuthenticationMethod}") - private String clientAuthenticationMethod; - - @Value("${security.oauth2.resource.userInfoUri}") - private String userInfoUri; - - @Bean - public ClientRegistrationRepository clientRegistrationRepository() { - ClientRegistration registration = ClientRegistration.withRegistrationId(registrationId) - .clientId(clientId) - .authorizationUri(authorizationUri) - .clientSecret(clientSecret) - .tokenUri(accessTokenUri) - .redirectUriTemplate(redirectUriTemplate) - .scope(scope.split(",")) - .clientName(clientName) - .authorizationGrantType(new AuthorizationGrantType(authorizationGrantType)) - .userInfoUri(userInfoUri) - .userNameAttributeName(userNameAttributeName) - .jwkSetUri(jwkSetUri) - .clientAuthenticationMethod(new ClientAuthenticationMethod(clientAuthenticationMethod)) - .build(); - return new InMemoryClientRegistrationRepository(Collections.singletonList(registration)); - } -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 36490b5166..beca0c37bb 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -18,8 +18,6 @@ package org.thingsboard.server.config; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Required; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.security.SecurityProperties; import org.springframework.context.annotation.Bean; @@ -41,6 +39,7 @@ import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; import org.thingsboard.server.service.security.auth.jwt.JwtTokenAuthenticationProcessingFilter; @@ -89,10 +88,7 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt @Autowired private JwtAuthenticationProvider jwtAuthenticationProvider; @Autowired private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider; - @Value("${security.oauth2.enabled}") - private boolean oauth2Enabled; - @Value("${security.oauth2.client.loginProcessingUrl}") - private String loginProcessingUrl; + @Autowired(required = false) OAuth2Configuration oauth2Configuration; @Autowired @Qualifier("jwtHeaderTokenExtractor") @@ -204,10 +200,12 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); - if (oauth2Enabled) { + if (oauth2Configuration.isEnabled()) { http.oauth2Login() - .loginProcessingUrl(loginProcessingUrl) + .loginPage("/oauth2Login") + .loginProcessingUrl(oauth2Configuration.getClients().values().iterator().next().getLoginProcessingUrl()) .successHandler(oauth2AuthenticationSuccessHandler); +// .and().oauth2Login().loginProcessingUrl(); } } diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 42da043a91..3097b9c2b1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -38,8 +38,10 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; @@ -55,6 +57,7 @@ import ua_parser.Client; import javax.servlet.http.HttpServletRequest; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; @RestController @TbCoreComponent @@ -80,6 +83,9 @@ public class AuthController extends BaseController { @Autowired private AuditLogService auditLogService; + @Autowired + private OAuth2Service oauth2Service; + @PreAuthorize("isAuthenticated()") @RequestMapping(value = "/auth/user", method = RequestMethod.GET) public @ResponseBody User getUser() throws ThingsboardException { @@ -330,4 +336,13 @@ public class AuthController extends BaseController { } } + @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) + @ResponseBody + public List getOath2Clients() throws ThingsboardException { + try { + return oauth2Service.getOAuth2Clients(); + } catch (Exception e) { + throw handleException(e); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java deleted file mode 100644 index e99e5896e1..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.security.auth.oauth; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.security.authentication.InsufficientAuthenticationException; -import org.springframework.security.authentication.LockedException; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.userdetails.UsernameNotFoundException; -import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.model.UserPrincipal; -import org.thingsboard.server.service.security.model.token.JwtToken; -import org.thingsboard.server.service.security.model.token.JwtTokenFactory; -import org.thingsboard.server.service.security.system.SystemSecurityService; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -@Component(value="oauth2AuthenticationSuccessHandler") -@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") -public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { - - private final ObjectMapper mapper; - private final JwtTokenFactory tokenFactory; - private final RefreshTokenRepository refreshTokenRepository; - private final SystemSecurityService systemSecurityService; - private final UserService userService; - - @Autowired - public Oauth2AuthenticationSuccessHandler(final ObjectMapper mapper, - final JwtTokenFactory tokenFactory, - final RefreshTokenRepository refreshTokenRepository, - final UserService userService, - final SystemSecurityService systemSecurityService) { - this.mapper = mapper; - this.tokenFactory = tokenFactory; - this.refreshTokenRepository = refreshTokenRepository; - this.userService = userService; - this.systemSecurityService = systemSecurityService; - } - - @Override - public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { - Object object = authentication.getPrincipal(); - - System.out.println(object); - - // active user check - - UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, "tenant@thingsboard.org"); - SecurityUser securityUser = (SecurityUser) authenticateByUsernameAndPassword(principal,"tenant@thingsboard.org", "tenant").getPrincipal(); - - JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); - JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); - - Map tokenMap = new HashMap(); - tokenMap.put("token", accessToken.getToken()); - tokenMap.put("refreshToken", refreshToken.getToken()); - -// response.setStatus(HttpStatus.OK.value()); -// response.setContentType(MediaType.APPLICATION_JSON_VALUE); -// mapper.writeValue(response.getWriter(), tokenMap); - - request.setAttribute("token", accessToken.getToken()); - response.addHeader("token", accessToken.getToken()); - - getRedirectStrategy().sendRedirect(request, response, "http://localhost:4200/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); - } - - private Authentication authenticateByUsernameAndPassword(UserPrincipal userPrincipal, String username, String password) { - User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, username); - if (user == null) { - throw new UsernameNotFoundException("User not found: " + username); - } - - try { - - UserCredentials userCredentials = userService.findUserCredentialsByUserId(TenantId.SYS_TENANT_ID, user.getId()); - if (userCredentials == null) { - throw new UsernameNotFoundException("User credentials not found"); - } - - try { - systemSecurityService.validateUserCredentials(user.getTenantId(), userCredentials, username, password); - } catch (LockedException e) { - throw e; - } - - if (user.getAuthority() == null) - throw new InsufficientAuthenticationException("User has no authority assigned"); - - SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), userPrincipal); - return new UsernamePasswordAuthenticationToken(securityUser, null, securityUser.getAuthorities()); - } catch (Exception e) { - throw e; - } - } -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java new file mode 100644 index 0000000000..10334a8430 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.TextPageLink; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public abstract class BaseOAuth2ClientMapper { + + @Autowired + private UserService userService; + + @Autowired + private TenantService tenantService; + + @Autowired + private CustomerService customerService; + + protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, boolean allowUserCreation) { + UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, oauth2User.getEmail()); + + User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, oauth2User.getEmail()); + + if (user == null && !allowUserCreation) { + throw new UsernameNotFoundException("User not found: " + oauth2User.getEmail()); + } + + if (user == null) { + user = new User(); + if (StringUtils.isEmpty(oauth2User.getCustomerName())) { + user.setAuthority(Authority.TENANT_ADMIN); + } else { + user.setAuthority(Authority.CUSTOMER_USER); + } + user.setTenantId(getTenantId(oauth2User.getTenantName())); + user.setCustomerId(getCustomerId(user.getTenantId(), oauth2User.getCustomerName())); + user.setEmail(oauth2User.getEmail()); + user.setFirstName(oauth2User.getFirstName()); + user.setLastName(oauth2User.getLastName()); + user = userService.saveUser(user); + } + + try { + SecurityUser securityUser = new SecurityUser(user, true, principal); + return (SecurityUser) new UsernamePasswordAuthenticationToken(securityUser, null, securityUser.getAuthorities()).getPrincipal(); + } catch (Exception e) { + log.error("Can't get or create security user from oauth2 user", e); + throw e; + } + } + + private TenantId getTenantId(String tenantName) { + List tenants = tenantService.findTenants(new TextPageLink(1, tenantName)).getData(); + Tenant tenant; + if (tenants == null || tenants.isEmpty()) { + tenant = new Tenant(); + tenant.setTitle(tenantName); + tenant = tenantService.saveTenant(tenant); + } else { + tenant = tenants.get(0); + } + return tenant.getTenantId(); + } + + private CustomerId getCustomerId(TenantId tenantId, String customerName) { + if (StringUtils.isEmpty(customerName)) { + return null; + } + Optional customerOpt = customerService.findCustomerByTenantIdAndTitle(tenantId, customerName); + if (customerOpt.isPresent()) { + return customerOpt.get().getId(); + } else { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(customerName); + return customerService.saveCustomer(customer).getId(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java new file mode 100644 index 0000000000..2e87d57c72 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.text.StrSubstitutor; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.Map; + +@Service(value = "basicOAuth2ClientMapper") +@Slf4j +public class BasicOAuth2ClientMapper extends BaseOAuth2ClientMapper implements OAuth2ClientMapper { + + @Override + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { + OAuth2User oauth2User = new OAuth2User(); + Map attributes = token.getPrincipal().getAttributes(); + String email = getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); + oauth2User.setEmail(email); + oauth2User.setTenantName(getTenantName(attributes, config)); + if (!StringUtils.isEmpty(config.getBasic().getLastNameAttributeKey())) { + String lastName = getStringAttributeByKey(attributes, config.getBasic().getLastNameAttributeKey()); + oauth2User.setLastName(lastName); + } + if (!StringUtils.isEmpty(config.getBasic().getFirstNameAttributeKey())) { + String firstName = getStringAttributeByKey(attributes, config.getBasic().getFirstNameAttributeKey()); + oauth2User.setFirstName(firstName); + } + if (!StringUtils.isEmpty(config.getBasic().getCustomerNameStrategyPattern())) { + StrSubstitutor sub = new StrSubstitutor(attributes, "${", "}"); + String customerName = sub.replace(config.getBasic().getCustomerNameStrategyPattern()); + oauth2User.setCustomerName(customerName); + } + return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); + } + + private String getTenantName(Map attributes, OAuth2ClientMapperConfig config) { + switch (config.getBasic().getTenantNameStrategy()) { + case "domain": + String email = getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); + return email.substring(email .indexOf("@") + 1); + case "custom": + StrSubstitutor sub = new StrSubstitutor(attributes, "${", "}"); + return sub.replace(config.getBasic().getTenantNameStrategyPattern()); + default: + throw new RuntimeException("Tenant Name Strategy with type " + config.getBasic().getTenantNameStrategy() + " is not supported!"); + } + } + + private String getStringAttributeByKey(Map attributes, String key) { + String result = null; + try { + result = (String) attributes.get(key); + + } catch (Exception e) { + log.warn("Can't convert attribute to String by key " + key); + } + return result; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java new file mode 100644 index 0000000000..cada6a7958 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; +import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.service.security.model.SecurityUser; + +@Service(value = "customOAuth2ClientMapper") +@Slf4j +public class CustomOAuth2ClientMapper extends BaseOAuth2ClientMapper implements OAuth2ClientMapper { + + private RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder(); + + @Override + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { + OAuth2User oauth2User = getOAuth2User(token, config.getCustom()); + return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); + } + + public OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) { + if (!StringUtils.isEmpty(custom.getUsername()) && !StringUtils.isEmpty(custom.getPassword())) { + restTemplateBuilder = restTemplateBuilder.basicAuthentication(custom.getUsername(), custom.getPassword()); + } + RestTemplate restTemplate = restTemplateBuilder.build(); + return restTemplate.postForEntity(custom.getUrl(), token.getPrincipal(), OAuth2User.class).getBody(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java new file mode 100644 index 0000000000..196bfe7b50 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface OAuth2ClientMapper { + SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config); +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java new file mode 100644 index 0000000000..e1c5b694bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class OAuth2ClientMapperProvider { + + @Autowired + @Qualifier("basicOAuth2ClientMapper") + private OAuth2ClientMapper basicOAuth2ClientMapper; + + @Autowired + @Qualifier("customOAuth2ClientMapper") + private OAuth2ClientMapper customOAuth2ClientMapper; + + public OAuth2ClientMapper getOAuth2ClientMapperByType(String oauth2ClientType) { + switch (oauth2ClientType) { + case "custom": + return customOAuth2ClientMapper; + case "basic": + return basicOAuth2ClientMapper; + default: + throw new RuntimeException("OAuth2ClientMapper with type " + oauth2ClientType + " is not supported!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java new file mode 100644 index 0000000000..be8f7ca7c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.oauth2.OAuth2Client; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; +import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.token.JwtToken; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +@Component(value = "oauth2AuthenticationSuccessHandler") +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { + + private final JwtTokenFactory tokenFactory; + private final RefreshTokenRepository refreshTokenRepository; + private final OAuth2ClientMapperProvider oauth2ClientMapperProvider; + private final OAuth2Configuration oauth2Configuration; + + @Autowired + public Oauth2AuthenticationSuccessHandler(final JwtTokenFactory tokenFactory, + final RefreshTokenRepository refreshTokenRepository, + final OAuth2ClientMapperProvider oauth2ClientMapperProvider, + final OAuth2Configuration oauth2Configuration) { + this.tokenFactory = tokenFactory; + this.refreshTokenRepository = refreshTokenRepository; + this.oauth2ClientMapperProvider = oauth2ClientMapperProvider; + this.oauth2Configuration = oauth2Configuration; + } + + @Override + public void onAuthenticationSuccess(HttpServletRequest request, + HttpServletResponse response, + Authentication authentication) throws IOException { + OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication; + + OAuth2Client oauth2Client = oauth2Configuration.getClientByRegistrationId(token.getAuthorizedClientRegistrationId()); + OAuth2ClientMapper mapper = oauth2ClientMapperProvider.getOAuth2ClientMapperByType(oauth2Client.getMapperConfig().getType()); + SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(token, oauth2Client.getMapperConfig()); + + JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); + JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); + + getRedirectStrategy().sendRedirect(request, response, "/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index d26b02174e..4983071c5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -@Component(value="defaultAuthenticationSuccessHandler") +@Component(value = "defaultAuthenticationSuccessHandler") public class RestAwareAuthenticationSuccessHandler implements AuthenticationSuccessHandler { private final ObjectMapper mapper; private final JwtTokenFactory tokenFactory; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 97e22fa2c6..e2808bc8f6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -99,29 +99,13 @@ security: duration: "${SECURITY_CLAIM_DURATION:60000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value basic: enabled: false - # oauth2: - # enabled: true - # registrationId: A - # userNameAttributeName: email - # client: - # clientName: Thingsboard Dev Test Q - # clientId: 5f5c0998-1d9b-4679-9610-6108fb91af2a - # clientSecret: h_kXVb7Ee1LgDDinix_nkAh_owWX7YCO783NNteF9AIOqlTWu2L03YoFjv5KL8yRVyx4uYAE-r_N3tFbupE8Kw - # accessTokenUri: https://federation-q.auth.schwarz/nidp/oauth/nam/token - # authorizationUri: https://federation-q.auth.schwarz/nidp/oauth/nam/authz - # scope: openid,profile,email,siam - # redirectUriTemplate: http://localhost:8080/login/oauth2/code/ - # loginProcessingUrl: /login/oauth2/code/ - # jwkSetUri: https://federation-q.auth.schwarz/nidp/oauth/nam/keys - # authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials - # clientAuthenticationMethod: post # basic, post - # resource: - # userInfoUri: https://federation-q.auth.schwarz/nidp/oauth/nam/userinfo - oauth2: - enabled: true - registrationId: A - userNameAttributeName: email - client: + oauth2: + enabled: true + clients: + schwarz: + registrationId: A + loginButtonLabel: Auth0 # + loginButtonIcon: clientName: Test app clientId: dVH9reqyqiXIG7M2wmamb0ySue8zaM4g clientSecret: EYAfAGxwkwoeYnb2o2cDgaWZB5k97OStpZQPPvcMMD-SVH2BuughTGeBazXtF5I6 @@ -133,8 +117,53 @@ security: jwkSetUri: https://dev-r9m8ht0k.auth0.com/.well-known/jwks.json authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials clientAuthenticationMethod: post # basic, post - resource: userInfoUri: https://dev-r9m8ht0k.auth0.com/userinfo + userNameAttributeName: email + mapperConfig: + type: custom # basic or custom + basic: + allowUserCreation: true # required + emailAttributeKey: email # required + firstNameAttributeKey: + lastNameAttributeKey: + tenantNameStrategy: domain # domain or custom + tenantNameStrategyPattern: + customerNameStrategyPattern: + custom: + url: http://localhost:9090/oauth2/mapper + username: admin + password: bababa + auth0: + registrationId: B + loginButtonLabel: Schwarz # + loginButtonIcon: mdi:google + clientName: Thingsboard Dev Test Q + clientId: 5f5c0998-1d9b-4679-9610-6108fb91af2a + clientSecret: h_kXVb7Ee1LgDDinix_nkAh_owWX7YCO783NNteF9AIOqlTWu2L03YoFjv5KL8yRVyx4uYAE-r_N3tFbupE8Kw + accessTokenUri: https://federation-q.auth.schwarz/nidp/oauth/nam/token + authorizationUri: https://federation-q.auth.schwarz/nidp/oauth/nam/authz + scope: openid,profile,email,siam + redirectUriTemplate: http://localhost:8080/login/oauth2/code/ + loginProcessingUrl: /login/oauth2/code/ + jwkSetUri: https://federation-q.auth.schwarz/nidp/oauth/nam/keys + authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials + clientAuthenticationMethod: post # basic, post + userInfoUri: https://federation-q.auth.schwarz/nidp/oauth/nam/userinfo + userNameAttributeName: mail + mapperConfig: + type: basic # simple or custom + basic: + allowUserCreation: true # required + emailAttributeKey: CloudLoginName # required + firstNameAttributeKey: givenName + lastNameAttributeKey: sn + tenantNameStrategy: custom # domain or custom + tenantNameStrategyPattern: LOL ${region} + customerNameStrategyPattern: GGG ${countrycode} + custom: + url: http://localhost:9090/oauth2/mapper + username: test + password: test # Dashboard parameters dashboard: diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java new file mode 100644 index 0000000000..d72b6ef98c --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; + +import java.util.List; + +public interface OAuth2Service { + + List getOAuth2Clients(); +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java new file mode 100644 index 0000000000..6337171369 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; + +@Data +public class OAuth2User { + private String tenantName; + private String customerName; + private String email; + private String firstName; + private String lastName; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java new file mode 100644 index 0000000000..30fd55d204 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.UUID; + +public class OAuth2IntegrationId extends UUIDBased { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public OAuth2IntegrationId(@JsonProperty("id") UUID id) { + super(id); + } + + public static OAuth2IntegrationId fromString(String oauth2IntegrationId) { + return new OAuth2IntegrationId(UUID.fromString(oauth2IntegrationId)); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java new file mode 100644 index 0000000000..0ee5832e63 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.OAuth2IntegrationId; + +@EqualsAndHashCode(callSuper = true) +@Data +public class OAuth2ClientInfo extends BaseData { + + private String name; + private String icon; + private String url; + + public OAuth2ClientInfo() { + super(); + } + + public OAuth2ClientInfo(OAuth2IntegrationId id) { + super(id); + } + + public OAuth2ClientInfo(OAuth2ClientInfo oauth2ClientInfo) { + super(oauth2ClientInfo); + this.name = oauth2ClientInfo.getName(); + this.icon = oauth2ClientInfo.getIcon(); + this.url = oauth2ClientInfo.getUrl(); + } +} diff --git a/dao/pom.xml b/dao/pom.xml index ca81ee996c..ebee947f7a 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -115,6 +115,10 @@ org.springframework spring-web provided + + + org.springframework.security + spring-security-oauth2-client com.datastax.cassandra diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java new file mode 100644 index 0000000000..1327e418cc --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; + +@Data +public class OAuth2Client { + + private String registrationId; + private String loginButtonLabel; + private String loginButtonIcon; + private String clientName; + private String clientId; + private String clientSecret; + private String accessTokenUri; + private String authorizationUri; + private String scope; + private String redirectUriTemplate; + private String jwkSetUri; + private String loginProcessingUrl; + private String authorizationGrantType; + private String clientAuthenticationMethod; + private String userInfoUri; + private String userNameAttributeName; + private OAuth2ClientMapperConfig mapperConfig; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java new file mode 100644 index 0000000000..2c8b7cbfc7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; + +@Data +public class OAuth2ClientMapperConfig { + + private String type; + private CustomOAuth2ClientMapperConfig custom; + private BasicOAuth2ClientMapperConfig basic; + + @Data + public static class BasicOAuth2ClientMapperConfig { + private boolean allowUserCreation; + private String emailAttributeKey; + private String firstNameAttributeKey; + private String lastNameAttributeKey; + private String tenantNameStrategy; + private String tenantNameStrategyPattern; + private String customerNameStrategyPattern; + } + + @Data + public static class CustomOAuth2ClientMapperConfig { + private String url; + private String username; + private String password; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java new file mode 100644 index 0000000000..fa51121d75 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.oauth2.client.registration.ClientRegistration; +import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; +import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; +import org.springframework.security.oauth2.core.AuthorizationGrantType; +import org.springframework.security.oauth2.core.ClientAuthenticationMethod; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Configuration +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true", matchIfMissing = true) +@ConfigurationProperties(prefix = "security.oauth2") +@Data +@Slf4j +public class OAuth2Configuration { + + private boolean enabled; + private Map clients = new HashMap<>(); + + @Bean + public ClientRegistrationRepository clientRegistrationRepository() { + List result = new ArrayList<>(); + for (OAuth2Client client : clients.values()) { + ClientRegistration registration = ClientRegistration.withRegistrationId(client.getRegistrationId()) + .clientId(client.getClientId()) + .authorizationUri(client.getAuthorizationUri()) + .clientSecret(client.getClientSecret()) + .tokenUri(client.getAccessTokenUri()) + .redirectUriTemplate(client.getRedirectUriTemplate()) + .scope(client.getScope().split(",")) + .clientName(client.getClientName()) + .authorizationGrantType(new AuthorizationGrantType(client.getAuthorizationGrantType())) + .userInfoUri(client.getUserInfoUri()) + .userNameAttributeName(client.getUserNameAttributeName()) + .jwkSetUri(client.getJwkSetUri()) + .clientAuthenticationMethod(new ClientAuthenticationMethod(client.getClientAuthenticationMethod())) + .build(); + result.add(registration); + } + return new InMemoryClientRegistrationRepository(result); + } + + public OAuth2Client getClientByRegistrationId(String registrationId) { + OAuth2Client result = null; + if (clients != null && !clients.isEmpty()) { + for (OAuth2Client client : clients.values()) { + if (client.getRegistrationId().equals(registrationId)) { + result = client; + break; + } + } + } + return result; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java new file mode 100644 index 0000000000..14aa390259 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Slf4j +@Service +public class OAuth2ServiceImpl implements OAuth2Service { + + @Autowired(required = false) + OAuth2Configuration oauth2Configuration; + + @Override + public List getOAuth2Clients() { + if (!oauth2Configuration.isEnabled()) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + for (OAuth2Client c : oauth2Configuration.getClients().values()) { + OAuth2ClientInfo client = new OAuth2ClientInfo(); + client.setName(c.getLoginButtonLabel()); + client.setUrl(String.format("/oauth2/authorization/%s", c.getRegistrationId())); + client.setIcon(c.getLoginButtonIcon()); + result.add(client); + } + return result; + } +} diff --git a/pom.xml b/pom.xml index f263307538..0ea3016b78 100755 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ ${basedir} thingsboard 2.2.4.RELEASE + 2.1.2.RELEASE 5.2.2.RELEASE 5.2.2.RELEASE 2.2.4.RELEASE @@ -461,7 +462,7 @@ org.springframework.cloud spring-cloud-starter-oauth2 - ${spring-boot.version} + ${spring-oauth2.version} org.springframework.security diff --git a/ui/src/app/api/login.service.js b/ui/src/app/api/login.service.js index 0707070ed5..292268642f 100644 --- a/ui/src/app/api/login.service.js +++ b/ui/src/app/api/login.service.js @@ -18,7 +18,7 @@ export default angular.module('thingsboard.api.login', []) .name; /*@ngInject*/ -function LoginService($http, $q) { +function LoginService($http, $q, $rootScope) { var service = { activate: activate, @@ -28,6 +28,7 @@ function LoginService($http, $q) { publicLogin: publicLogin, resetPassword: resetPassword, sendResetPasswordLink: sendResetPasswordLink, + loadOAuth2Clients: loadOAuth2Clients } return service; @@ -109,4 +110,16 @@ function LoginService($http, $q) { }); return deferred.promise; } + + function loadOAuth2Clients(){ + var deferred = $q.defer(); + var url = '/api/noauth/oauth2Clients'; + $http.post(url).then(function success(response) { + $rootScope.oauth2Clients = response.data; + deferred.resolve(); + }, function fail() { + deferred.reject(); + }); + return deferred.promise; + } } diff --git a/ui/src/app/app.run.js b/ui/src/app/app.run.js index 5e56a83eaf..3255c0a917 100644 --- a/ui/src/app/app.run.js +++ b/ui/src/app/app.run.js @@ -17,7 +17,7 @@ import Flow from '@flowjs/ng-flow/dist/ng-flow-standalone.min'; import UrlHandler from './url.handler'; /*@ngInject*/ -export default function AppRun($rootScope, $window, $injector, $location, $log, $state, $mdDialog, $filter, loginService, userService, $translate) { +export default function AppRun($rootScope, $window, $injector, $location, $log, $state, $mdDialog, $filter, $q, loginService, userService, $translate) { $window.Flow = Flow; var frame = null; @@ -41,11 +41,13 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, } initWatchers(); - + + var skipStateChange = false; + function initWatchers() { $rootScope.unauthenticatedHandle = $rootScope.$on('unauthenticated', function (event, doLogout) { if (doLogout) { - $state.go('login'); + gotoPublicModule('login'); } else { UrlHandler($injector, $location); } @@ -61,6 +63,11 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, $rootScope.stateChangeStartHandle = $rootScope.$on('$stateChangeStart', function (evt, to, params) { + if (skipStateChange) { + skipStateChange = false; + return; + } + function waitForUserLoaded() { if ($rootScope.userLoadedHandle) { $rootScope.userLoadedHandle(); @@ -128,7 +135,10 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, redirectParams.toName = to.name; redirectParams.params = params; userService.setRedirectParams(redirectParams); - $state.go('login', params); + gotoPublicModule('login', params); + } else { + evt.preventDefault(); + gotoPublicModule(to.name, params); } } } else { @@ -158,6 +168,23 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, userService.gotoDefaultPlace(params); } + function gotoPublicModule(name, params) { + let tasks = []; + if (name === "login") { + tasks.push(loginService.loadOAuth2Clients()); + } + $q.all(tasks).then( + () => { + skipStateChange = true; + $state.go(name, params); + }, + () => { + skipStateChange = true; + $state.go(name, params); + } + ); + } + function showForbiddenDialog() { if (forbiddenDialog === null) { $translate(['access.access-forbidden', diff --git a/ui/src/app/locale/locale.constant-cs_CZ.json b/ui/src/app/locale/locale.constant-cs_CZ.json index 2a297cd9bb..13247f188a 100644 --- a/ui/src/app/locale/locale.constant-cs_CZ.json +++ b/ui/src/app/locale/locale.constant-cs_CZ.json @@ -1136,7 +1136,7 @@ "total": "celkem" }, "login": { - "login": "Přihlásit", + "login": "Přihlásit se", "request-password-reset": "Vyžádat reset hesla", "reset-password": "Reset hesla", "create-password": "Vytvořit heslo", @@ -1150,7 +1150,9 @@ "new-password": "Nové heslo", "new-password-again": "Nové heslo znovu", "password-link-sent-message": "Odkaz pro reset hesla byl úspěšně odeslán!", - "email": "Email" + "email": "Email", + "login-with": "Přihlásit se přes {{name}}", + "or": "nebo" }, "position": { "top": "Nahoře", diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index fa4689fc75..521669fd02 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1317,7 +1317,7 @@ } }, "login": { - "login": "Login", + "login": "Log in", "request-password-reset": "Request Password Reset", "reset-password": "Reset Password", "create-password": "Create Password", @@ -1332,7 +1332,9 @@ "new-password": "New password", "new-password-again": "New password again", "password-link-sent-message": "Password reset link was successfully sent!", - "email": "Email" + "email": "Email", + "login-with": "Login with {{name}}", + "or": "or" }, "position": { "top": "Top", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index e93c92124c..473698a091 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1246,7 +1246,9 @@ "new-password": "Новый пароль", "new-password-again": "Повторите новый пароль", "password-link-sent-message": "Ссылка для сброса пароля была успешно отправлена!", - "email": "Эл. адрес" + "email": "Эл. адрес", + "login-with": "Войти через {{name}}", + "or": "или" }, "position": { "top": "Верх", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index c19049bc11..79c2ece00f 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1646,7 +1646,7 @@ } }, "login": { - "login": "Вхід", + "login": "Увійти", "request-password-reset": "Запит скидання пароля", "reset-password": "Скинути пароль", "create-password": "Створити пароль", @@ -1661,7 +1661,9 @@ "new-password": "Новий пароль", "new-password-again": "Повторіть новий пароль", "password-link-sent-message": "Посилання для скидання пароля було успішно надіслано!", - "email": "Електронна пошта" + "email": "Електронна пошта", + "login-with": "Увійти через {{name}}", + "or": "або" }, "position": { "top": "Угорі", diff --git a/ui/src/app/login/login.scss b/ui/src/app/login/login.scss index a83b9c09ac..16520544fb 100644 --- a/ui/src/app/login/login.scss +++ b/ui/src/app/login/login.scss @@ -22,6 +22,10 @@ md-card.tb-login-card { width: 450px !important; } + .tb-padding { + padding: 8px; + } + md-card-title { img.tb-login-logo { height: 50px; @@ -31,4 +35,36 @@ md-card.tb-login-card { md-card-content { margin-top: -50px; } + + md-input-container .md-errors-spacer { + display: none; + } + + .oauth-container{ + .container-divider { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + width: 100%; + margin: 10px 0; + + .line { + flex: 1; + } + + .text { + padding-right: 10px; + padding-left: 10px; + } + } + + .material-icons{ + width: 20px; + min-width: 20px; + height: 20px; + min-height: 20px; + margin: 0 4px; + } + } } diff --git a/ui/src/app/login/login.tpl.html b/ui/src/app/login/login.tpl.html index 409f5cd4ad..e4f7f2559e 100644 --- a/ui/src/app/login/login.tpl.html +++ b/ui/src/app/login/login.tpl.html @@ -24,7 +24,7 @@ md-mode="indeterminate" ng-disabled="!$root.loading" ng-show="$root.loading"> - {{ 'login.login-with' | translate: {name: oauth2Client.name} }} From 9b18c408b2e2104d55df8d0c36ec9b68b2870641 Mon Sep 17 00:00:00 2001 From: VoBa Date: Wed, 6 May 2020 19:20:18 +0300 Subject: [PATCH 190/196] Added activate user config. Fixed https issue (#2737) * Added activate user config. Fixed https issue * Update OAuth2User.java Co-authored-by: Igor Kulikov --- .../ThingsboardSecurityConfiguration.java | 4 +-- .../server/controller/AuthController.java | 7 ++-- .../server/controller/BaseController.java | 33 ------------------ .../server/controller/UserController.java | 7 ++-- .../oauth2/AbstractOAuth2ClientMapper.java | 8 +++-- .../auth/oauth2/BasicOAuth2ClientMapper.java | 3 +- .../auth/oauth2/CustomOAuth2ClientMapper.java | 2 +- .../Oauth2AuthenticationSuccessHandler.java | 4 ++- .../thingsboard/server/utils/MiscUtils.java | 34 ++++++++++++++++++- .../src/main/resources/thingsboard.yml | 6 ++-- .../dao/oauth2/OAuth2ClientMapperConfig.java | 3 +- 11 files changed, 60 insertions(+), 51 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 9e967158b4..8a21fd229f 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -204,11 +204,11 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt http.oauth2Login() .loginPage("/oauth2Login") .loginProcessingUrl(oauth2Configuration.getLoginProcessingUrl()) - .successHandler(oauth2AuthenticationSuccessHandler); + .successHandler(oauth2AuthenticationSuccessHandler) + .failureHandler(failureHandler); } } - @Bean @ConditionalOnMissingBean(CorsFilter.class) public CorsFilter corsFilter(@Autowired MvcCorsProperties mvcCorsProperties) { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 49cfe482df..9f6c6f3b44 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.utils.MiscUtils; import ua_parser.Client; import javax.servlet.http.HttpServletRequest; @@ -170,7 +171,7 @@ public class AuthController extends BaseController { try { String email = resetPasswordByEmailRequest.get("email").asText(); UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl, userCredentials.getResetToken()); @@ -218,7 +219,7 @@ public class AuthController extends BaseController { User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); @@ -265,7 +266,7 @@ public class AuthController extends BaseController { User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId()); UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String loginUrl = String.format("%s/login", baseUrl); String email = user.getEmail(); mailService.sendPasswordWasResetEmail(loginUrl, email); diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 8385e61a20..a8d6098c4a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -521,39 +521,6 @@ public abstract class BaseController { return ruleNode; } - - protected String constructBaseUrl(HttpServletRequest request) { - String scheme = request.getScheme(); - - String forwardedProto = request.getHeader("x-forwarded-proto"); - if (forwardedProto != null) { - scheme = forwardedProto; - } - - int serverPort = request.getServerPort(); - if (request.getHeader("x-forwarded-port") != null) { - try { - serverPort = request.getIntHeader("x-forwarded-port"); - } catch (NumberFormatException e) { - } - } else if (forwardedProto != null) { - switch (forwardedProto) { - case "http": - serverPort = 80; - break; - case "https": - serverPort = 443; - break; - } - } - - String baseUrl = String.format("%s://%s:%d", - scheme, - request.getServerName(), - serverPort); - return baseUrl; - } - protected I emptyId(EntityType entityType) { return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index a37d9c29b6..bf64ad7d6f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; @@ -148,7 +149,7 @@ public class UserController extends BaseController { if (sendEmail) { SecurityUser authUser = getCurrentUser(); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId()); - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); String email = savedUser.getEmail(); @@ -188,7 +189,7 @@ public class UserController extends BaseController { UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId()); if (!userCredentials.isEnabled()) { - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); mailService.sendActivationEmail(activateUrl, email); @@ -213,7 +214,7 @@ public class UserController extends BaseController { SecurityUser authUser = getCurrentUser(); UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId()); if (!userCredentials.isEnabled()) { - String baseUrl = constructBaseUrl(request); + String baseUrl = MiscUtils.constructBaseUrl(request); String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, userCredentials.getActivateToken()); return activateUrl; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java index f620c34436..dd49418204 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java @@ -63,7 +63,7 @@ public abstract class AbstractOAuth2ClientMapper { private final Lock userCreationLock = new ReentrantLock(); - protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, boolean allowUserCreation) { + protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, boolean allowUserCreation, boolean activateUser) { UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, oauth2User.getEmail()); User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, oauth2User.getEmail()); @@ -93,8 +93,10 @@ public abstract class AbstractOAuth2ClientMapper { user.setFirstName(oauth2User.getFirstName()); user.setLastName(oauth2User.getLastName()); user = userService.saveUser(user); - UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId()); - userService.activateUserCredentials(user.getTenantId(), userCredentials.getActivateToken(), passwordEncoder.encode("")); + if (activateUser) { + UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId()); + userService.activateUserCredentials(user.getTenantId(), userCredentials.getActivateToken(), passwordEncoder.encode("")); + } } } catch (Exception e) { log.error("Can't get or create security user from oauth2 user", e); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java index 1d44a772aa..c6b6aeaae3 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java @@ -56,7 +56,8 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen String customerName = sub.replace(config.getBasic().getCustomerNamePattern()); oauth2User.setCustomerName(customerName); } - return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); + + return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.isAllowUserCreation(), config.isActivateUser()); } private String getTenantName(Map attributes, OAuth2ClientMapperConfig config) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java index 832a1cd39b..0fb4563987 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java @@ -38,7 +38,7 @@ public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper impleme @Override public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { OAuth2User oauth2User = getOAuth2User(token, config.getCustom()); - return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); + return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.isAllowUserCreation(), config.isActivateUser()); } private synchronized OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java index be8f7ca7c2..8702661196 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java @@ -27,6 +27,7 @@ import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtToken; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; +import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -65,6 +66,7 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); - getRedirectStrategy().sendRedirect(request, response, "/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); + String baseUrl = MiscUtils.constructBaseUrl(request); + getRedirectStrategy().sendRedirect(request, response, baseUrl + "/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); } } \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java b/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java index bec28a3bdb..ed13ca603d 100644 --- a/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java @@ -18,8 +18,8 @@ package org.thingsboard.server.utils; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import javax.servlet.http.HttpServletRequest; import java.nio.charset.Charset; -import java.util.Random; /** @@ -47,4 +47,36 @@ public class MiscUtils { throw new IllegalArgumentException("Can't find hash function with name " + name); } } + + public static String constructBaseUrl(HttpServletRequest request) { + String scheme = request.getScheme(); + + String forwardedProto = request.getHeader("x-forwarded-proto"); + if (forwardedProto != null) { + scheme = forwardedProto; + } + + int serverPort = request.getServerPort(); + if (request.getHeader("x-forwarded-port") != null) { + try { + serverPort = request.getIntHeader("x-forwarded-port"); + } catch (NumberFormatException e) { + } + } else if (forwardedProto != null) { + switch (forwardedProto) { + case "http": + serverPort = 80; + break; + case "https": + serverPort = 443; + break; + } + } + + String baseUrl = String.format("%s://%s:%d", + scheme, + request.getServerName(), + serverPort); + return baseUrl; + } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8010987a43..aa1d03d92c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -127,11 +127,13 @@ security: userInfoUri: "${SECURITY_OAUTH2_DEFAULT_USER_INFO_URI:}" userNameAttributeName: "${SECURITY_OAUTH2_DEFAULT_USER_NAME_ATTRIBUTE_NAME:email}" mapperConfig: + # Allows to create user if it not exists + allowUserCreation: "${SECURITY_OAUTH2_DEFAULT_MAPPER_ALLOW_USER_CREATION:true}" + # Allows user to setup ThingsBoard internal password and login over default Login window + activateUser: "${SECURITY_OAUTH2_DEFAULT_MAPPER_ACTIVATE_USER:false}" # Mapper type of converter from external user into internal - 'basic' or 'custom' type: "${SECURITY_OAUTH2_DEFAULT_MAPPER_TYPE:basic}" basic: - # Allows to create user if it not exists - allowUserCreation: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_ALLOW_USER_CREATION:true}" # Key from attributes of external user object to use as email emailAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_EMAIL_ATTRIBUTE_KEY:email}" firstNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_FIRST_NAME_ATTRIBUTE_KEY:}" diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java index 695bb0aa0e..47f3746980 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java @@ -20,13 +20,14 @@ import lombok.Data; @Data public class OAuth2ClientMapperConfig { + private boolean allowUserCreation; + private boolean activateUser; private String type; private BasicOAuth2ClientMapperConfig basic; private CustomOAuth2ClientMapperConfig custom; @Data public static class BasicOAuth2ClientMapperConfig { - private boolean allowUserCreation; private String emailAttributeKey; private String firstNameAttributeKey; private String lastNameAttributeKey; From d9bfd829260df7e313e92472e83a4f87e95d93b9 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 6 May 2020 20:04:09 +0300 Subject: [PATCH 191/196] Introduce OAuth failure handling --- .../ThingsboardSecurityConfiguration.java | 11 ++++- .../Oauth2AuthenticationFailureHandler.java | 43 +++++++++++++++++++ ...RestAwareAuthenticationFailureHandler.java | 2 +- ui/src/app/api/user.service.js | 17 +++++++- ui/src/app/locale/locale.constant-en_US.json | 3 +- 5 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 8a21fd229f..683345fe7f 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -79,11 +79,18 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt @Qualifier("oauth2AuthenticationSuccessHandler") private AuthenticationSuccessHandler oauth2AuthenticationSuccessHandler; + @Autowired(required = false) + @Qualifier("oauth2AuthenticationFailureHandler") + private AuthenticationFailureHandler oauth2AuthenticationFailureHandler; + @Autowired @Qualifier("defaultAuthenticationSuccessHandler") private AuthenticationSuccessHandler successHandler; - @Autowired private AuthenticationFailureHandler failureHandler; + @Autowired + @Qualifier("defaultAuthenticationFailureHandler") + private AuthenticationFailureHandler failureHandler; + @Autowired private RestAuthenticationProvider restAuthenticationProvider; @Autowired private JwtAuthenticationProvider jwtAuthenticationProvider; @Autowired private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider; @@ -205,7 +212,7 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .loginPage("/oauth2Login") .loginProcessingUrl(oauth2Configuration.getLoginProcessingUrl()) .successHandler(oauth2AuthenticationSuccessHandler) - .failureHandler(failureHandler); + .failureHandler(oauth2AuthenticationFailureHandler); } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java new file mode 100644 index 0000000000..653be85f72 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationFailureHandler.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler; +import org.springframework.stereotype.Component; +import org.thingsboard.server.utils.MiscUtils; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +@Component(value = "oauth2AuthenticationFailureHandler") +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler { + + @Override + public void onAuthenticationFailure(HttpServletRequest request, + HttpServletResponse response, AuthenticationException exception) + throws IOException, ServletException { + String baseUrl = MiscUtils.constructBaseUrl(request); + getRedirectStrategy().sendRedirect(request, response, baseUrl + "/login?loginError=" + + URLEncoder.encode(exception.getMessage(), StandardCharsets.UTF_8.toString())); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java index 726ee76a2d..38486e9a48 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationFailureHandler.java @@ -26,7 +26,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -@Component +@Component(value = "defaultAuthenticationFailureHandler") public class RestAwareAuthenticationFailureHandler implements AuthenticationFailureHandler { private final ThingsboardErrorResponseHandler errorResponseHandler; diff --git a/ui/src/app/api/user.service.js b/ui/src/app/api/user.service.js index 9ac8270403..b84bcf8428 100644 --- a/ui/src/app/api/user.service.js +++ b/ui/src/app/api/user.service.js @@ -22,7 +22,7 @@ export default angular.module('thingsboard.api.user', [thingsboardApiLogin, .name; /*@ngInject*/ -function UserService($http, $q, $rootScope, adminService, dashboardService, timeService, loginService, toast, store, jwtHelper, $translate, $state, $location) { +function UserService($http, $q, $rootScope, adminService, dashboardService, timeService, loginService, toast, store, jwtHelper, $translate, $state, $location, $mdDialog) { var currentUser = null, currentUserDetails = null, lastPublicDashboardId = null, @@ -406,6 +406,10 @@ function UserService($http, $q, $rootScope, adminService, dashboardService, time }, function fail() { deferred.reject(); }); + } else if (locationSearch.loginError) { + showLoginErrorDialog(locationSearch.loginError); + $location.search('loginError', null); + deferred.reject(); } else { procceedJwtTokenValidate(); } @@ -415,6 +419,17 @@ function UserService($http, $q, $rootScope, adminService, dashboardService, time return deferred.promise; } + function showLoginErrorDialog(loginError) { + $translate(['login.error', + 'action.close']).then(function (translations) { + var alert = $mdDialog.alert() + .title(translations['login.error']) + .htmlContent(loginError) + .ok(translations['action.close']); + $mdDialog.show(alert); + }); + } + function loadIsUserTokenAccessEnabled() { var deferred = $q.defer(); if (currentUser.authority === 'SYS_ADMIN' || currentUser.authority === 'TENANT_ADMIN') { diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index 6693875edf..5a8a588446 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1334,7 +1334,8 @@ "password-link-sent-message": "Password reset link was successfully sent!", "email": "Email", "login-with": "Login with {{name}}", - "or": "or" + "or": "or", + "error": "Login error" }, "position": { "top": "Top", From 71f94358c4744060482695dae7a49cd0c2415238 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 7 May 2020 10:49:44 +0300 Subject: [PATCH 192/196] Fix typos and added break statement --- application/src/main/resources/thingsboard.yml | 4 ++-- .../server/common/transport/adaptor/JsonConverter.java | 1 + .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03e461015f..2156447005 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -240,11 +240,11 @@ sql: ttl: ts: enabled: "${SQL_TTL_TS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of miliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 56e2c87d6e..8375b84ffa 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -391,6 +391,7 @@ public class JsonConverter { break; case JSON_V: result.add(de.getKv().getKey(), JSON_PARSER.parse(de.getKv().getJsonV())); + break; default: throw new IllegalArgumentException("Unsupported data type: " + de.getKv().getType()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 8aa15a4f38..9c46cbbc1d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -177,6 +177,7 @@ public abstract class TbAbstractGetAttributesNode Date: Thu, 7 May 2020 11:19:27 +0300 Subject: [PATCH 193/196] Fix error update websocket to change user (#2743) --- ui/src/app/api/telemetry-websocket.service.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/src/app/api/telemetry-websocket.service.js b/ui/src/app/api/telemetry-websocket.service.js index dc1f358aed..edc1fb3147 100644 --- a/ui/src/app/api/telemetry-websocket.service.js +++ b/ui/src/app/api/telemetry-websocket.service.js @@ -74,6 +74,10 @@ function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, $m } }); + $rootScope.telemetryWsLoginHandle = $rootScope.$on('authenticated', function () { + reset(true); + }); + return service; function publishCommands () { From f2757e0535bdab147e9148b393b12c1d0f0a4c6b Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 7 May 2020 14:21:17 +0300 Subject: [PATCH 194/196] Use source ip load balancing algorithm for http requests --- docker/haproxy/config/haproxy.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/haproxy/config/haproxy.cfg b/docker/haproxy/config/haproxy.cfg index bc36364024..dfe49247f5 100644 --- a/docker/haproxy/config/haproxy.cfg +++ b/docker/haproxy/config/haproxy.cfg @@ -108,7 +108,7 @@ backend tb-http-backend server tbHttp2 tb-http-transport2:8081 check inter 5s resolvers docker_resolver resolve-prefer ipv4 backend tb-api-backend - balance leastconn + balance source option tcp-check option log-health-checks server tbApi1 tb-core1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4 From 424d631a69c945387b7d16cec07cf9d407879303 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Fri, 8 May 2020 14:38:04 +0300 Subject: [PATCH 195/196] added extra properties to kafka consumer (#2751) * added extra properties to kafka consumer * added default values for kafka consumer properties in TbKafkaSettings * Update TbKafkaSettings.java * Update thingsboard.yml Co-authored-by: Andrew Shvayka --- application/src/main/resources/thingsboard.yml | 3 +++ .../server/queue/kafka/TbKafkaConsumerTemplate.java | 3 +++ .../server/queue/kafka/TbKafkaSettings.java | 12 ++++++++++++ 3 files changed, 18 insertions(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f565c5ce44..ec9ca29468 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -584,6 +584,9 @@ queue: linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 1b5619fd26..de94db804d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -54,6 +54,9 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue if (groupId != null) { props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); } + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, settings.getMaxPollRecords()); + props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, settings.getMaxPartitionFetchBytes()); + props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, settings.getFetchMaxBytes()); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit); props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 66121cb215..659dd19bda 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -55,6 +55,18 @@ public class TbKafkaSettings { @Getter private short replicationFactor; + @Value("${queue.kafka.max_poll_records:8192}") + @Getter + private int maxPollRecords; + + @Value("${queue.kafka.max_partition_fetch_bytes:16777216}") + @Getter + private int maxPartitionFetchBytes; + + @Value("${queue.kafka.fetch_max_bytes:134217728}") + @Getter + private int fetchMaxBytes; + @Value("${kafka.other:#{null}}") private List other; From 7e66fd269332482b8f53597ddca23e4e1197606e Mon Sep 17 00:00:00 2001 From: VoBa Date: Fri, 8 May 2020 16:11:59 +0300 Subject: [PATCH 196/196] Fix for pem key with password (#2752) * Added activate user config. Fixed https issue * Added possibility to encode PEM files encrypted with a private key - Integration * Remove PE fixes --- .../credentials/CertPemClientCredentials.java | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/CertPemClientCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/CertPemClientCredentials.java index 7c140176b7..60895fa002 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/CertPemClientCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/credentials/CertPemClientCredentials.java @@ -21,7 +21,6 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.mqtt.MqttClientConfig; import org.apache.commons.codec.binary.Base64; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openssl.PEMDecryptorProvider; @@ -30,15 +29,26 @@ import org.bouncycastle.openssl.PEMKeyPair; import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder; import org.springframework.util.StringUtils; +import org.thingsboard.mqtt.MqttClientConfig; +import javax.crypto.Cipher; +import javax.crypto.EncryptedPrivateKeyInfo; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; import java.io.ByteArrayInputStream; -import java.security.*; +import java.security.AlgorithmParameters; +import java.security.Key; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.Security; import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; -import java.security.interfaces.RSAPrivateKey; +import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.util.Optional; @@ -138,16 +148,36 @@ public class CertPemClientCredentials implements MqttClientCredentials { } private PrivateKey readPrivateKeyFile(String fileContent) throws Exception { - RSAPrivateKey privateKey = null; + PrivateKey privateKey = null; if (fileContent != null && !fileContent.isEmpty()) { fileContent = fileContent.replaceAll(".*BEGIN.*PRIVATE KEY.*", "") .replaceAll(".*END.*PRIVATE KEY.*", "") .replaceAll("\\s", ""); byte[] decoded = Base64.decodeBase64(fileContent); KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - privateKey = (RSAPrivateKey) keyFactory.generatePrivate(new PKCS8EncodedKeySpec(decoded)); + KeySpec keySpec = getKeySpec(decoded); + privateKey = keyFactory.generatePrivate(keySpec); } return privateKey; } + private KeySpec getKeySpec(byte[] encodedKey) throws Exception { + KeySpec keySpec; + if (password == null) { + keySpec = new PKCS8EncodedKeySpec(encodedKey); + } else { + PBEKeySpec pbeKeySpec = new PBEKeySpec(password.toCharArray()); + + EncryptedPrivateKeyInfo privateKeyInfo = new EncryptedPrivateKeyInfo(encodedKey); + String algorithmName = privateKeyInfo.getAlgName(); + Cipher cipher = Cipher.getInstance(algorithmName); + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance(algorithmName); + + Key pbeKey = secretKeyFactory.generateSecret(pbeKeySpec); + AlgorithmParameters algParams = privateKeyInfo.getAlgParameters(); + cipher.init(Cipher.DECRYPT_MODE, pbeKey, algParams); + keySpec = privateKeyInfo.getKeySpec(cipher); + } + return keySpec; + } }